From ee13b3f35c7238eff323450599185a775d726462 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 8 Apr 2025 18:09:12 +0800 Subject: [PATCH 01/16] scripts/kernel-doc.py: move KernelFiles class to a separate file The KernelFiles class is the main dispatcher which parses each source file. In preparation for letting kerneldoc Sphinx extension to import Python libraries, move regex ancillary classes to a separate file. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/80bc855e128a9ff0a11df5afe9ba71775dfc9a0f.1744106241.git.mchehab+huawei@kernel.org --- scripts/kernel-doc.py | 220 +-------------------------- scripts/lib/kdoc/kdoc_files.py | 270 +++++++++++++++++++++++++++++++++ 2 files changed, 271 insertions(+), 219 deletions(-) create mode 100755 scripts/lib/kdoc/kdoc_files.py diff --git a/scripts/kernel-doc.py b/scripts/kernel-doc.py index f030a36a165b..d09ada2d862a 100755 --- a/scripts/kernel-doc.py +++ b/scripts/kernel-doc.py @@ -119,6 +119,7 @@ sys.path.insert(0, os.path.join(SRC_DIR, LIB_DIR)) from kdoc_parser import KernelDoc, type_param from kdoc_re import Re +from kdoc_files import KernelFiles function_pointer = Re(r"([^\(]*\(\*)\s*\)\s*\(([^\)]*)\)", cache=False) @@ -143,225 +144,6 @@ type_member = Re(r"\&([_\w]+)(\.|->)([_\w]+)", cache=False) type_fallback = Re(r"\&([_\w]+)", cache=False) type_member_func = type_member + Re(r"\(\)", cache=False) -class GlobSourceFiles: - """ - Parse C source code file names and directories via an Interactor. - - """ - - def __init__(self, srctree=None, valid_extensions=None): - """ - Initialize valid extensions with a tuple. - - If not defined, assume default C extensions (.c and .h) - - It would be possible to use python's glob function, but it is - very slow, and it is not interactive. So, it would wait to read all - directories before actually do something. - - So, let's use our own implementation. - """ - - if not valid_extensions: - self.extensions = (".c", ".h") - else: - self.extensions = valid_extensions - - self.srctree = srctree - - def _parse_dir(self, dirname): - """Internal function to parse files recursively""" - - with os.scandir(dirname) as obj: - for entry in obj: - name = os.path.join(dirname, entry.name) - - if entry.is_dir(): - yield from self._parse_dir(name) - - if not entry.is_file(): - continue - - basename = os.path.basename(name) - - if not basename.endswith(self.extensions): - continue - - yield name - - def parse_files(self, file_list, file_not_found_cb): - for fname in file_list: - if self.srctree: - f = os.path.join(self.srctree, fname) - else: - f = fname - - if os.path.isdir(f): - yield from self._parse_dir(f) - elif os.path.isfile(f): - yield f - elif file_not_found_cb: - file_not_found_cb(fname) - - -class KernelFiles(): - - def parse_file(self, fname): - - doc = KernelDoc(self.config, fname) - doc.run() - - return doc - - def process_export_file(self, fname): - try: - with open(fname, "r", encoding="utf8", - errors="backslashreplace") as fp: - for line in fp: - KernelDoc.process_export(self.config.function_table, line) - - except IOError: - print(f"Error: Cannot open fname {fname}", fname=sys.stderr) - self.config.errors += 1 - - def file_not_found_cb(self, fname): - self.config.log.error("Cannot find file %s", fname) - self.config.errors += 1 - - def __init__(self, files=None, verbose=False, out_style=None, - werror=False, wreturn=False, wshort_desc=False, - wcontents_before_sections=False, - logger=None, modulename=None, export_file=None): - """Initialize startup variables and parse all files""" - - - if not verbose: - verbose = bool(os.environ.get("KBUILD_VERBOSE", 0)) - - if not modulename: - modulename = "Kernel API" - - dt = datetime.now() - if os.environ.get("KBUILD_BUILD_TIMESTAMP", None): - # use UTC TZ - to_zone = tz.gettz('UTC') - dt = dt.astimezone(to_zone) - - if not werror: - kcflags = os.environ.get("KCFLAGS", None) - if kcflags: - match = re.search(r"(\s|^)-Werror(\s|$)/", kcflags) - if match: - werror = True - - # reading this variable is for backwards compat just in case - # someone was calling it with the variable from outside the - # kernel's build system - kdoc_werror = os.environ.get("KDOC_WERROR", None) - if kdoc_werror: - werror = kdoc_werror - - # Set global config data used on all files - self.config = argparse.Namespace - - self.config.verbose = verbose - self.config.werror = werror - self.config.wreturn = wreturn - self.config.wshort_desc = wshort_desc - self.config.wcontents_before_sections = wcontents_before_sections - self.config.modulename = modulename - - self.config.function_table = set() - self.config.source_map = {} - - if not logger: - self.config.log = logging.getLogger("kernel-doc") - else: - self.config.log = logger - - self.config.kernel_version = os.environ.get("KERNELVERSION", - "unknown kernel version'") - self.config.src_tree = os.environ.get("SRCTREE", None) - - self.out_style = out_style - self.export_file = export_file - - # Initialize internal variables - - self.config.errors = 0 - self.results = [] - - self.file_list = files - self.files = set() - - def parse(self): - """ - Parse all files - """ - - glob = GlobSourceFiles(srctree=self.config.src_tree) - - # Let's use a set here to avoid duplicating files - - for fname in glob.parse_files(self.file_list, self.file_not_found_cb): - if fname in self.files: - continue - - self.files.add(fname) - - res = self.parse_file(fname) - self.results.append((res.fname, res.entries)) - - if not self.files: - sys.exit(1) - - # If a list of export files was provided, parse EXPORT_SYMBOL* - # from the ones not already parsed - - if self.export_file: - files = self.files - - glob = GlobSourceFiles(srctree=self.config.src_tree) - - for fname in glob.parse_files(self.export_file, - self.file_not_found_cb): - if fname not in files: - files.add(fname) - - self.process_export_file(fname) - - def out_msg(self, fname, name, arg): - # TODO: filter out unwanted parts - - return self.out_style.msg(fname, name, arg) - - def msg(self, enable_lineno=False, export=False, internal=False, - symbol=None, nosymbol=None): - - function_table = self.config.function_table - - if symbol: - for s in symbol: - function_table.add(s) - - # Output none mode: only warnings will be shown - if not self.out_style: - return - - self.out_style.set_config(self.config) - - self.out_style.set_filter(export, internal, symbol, nosymbol, - function_table, enable_lineno) - - for fname, arg_tuple in self.results: - for name, arg in arg_tuple: - if self.out_msg(fname, name, arg): - ln = arg.get("ln", 0) - dtype = arg.get('type', "") - - self.config.log.warning("%s:%d Can't handle %s", - fname, ln, dtype) - class OutputFormat: # output mode. diff --git a/scripts/lib/kdoc/kdoc_files.py b/scripts/lib/kdoc/kdoc_files.py new file mode 100755 index 000000000000..8bcdc7ead984 --- /dev/null +++ b/scripts/lib/kdoc/kdoc_files.py @@ -0,0 +1,270 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +# Copyright(c) 2025: Mauro Carvalho Chehab . +# +# pylint: disable=R0903,R0913,R0914,R0917 + +# TODO: implement warning filtering + +""" +Parse lernel-doc tags on multiple kernel source files. +""" + +import argparse +import logging +import os +import re +import sys +from datetime import datetime + +from dateutil import tz + +from kdoc_parser import KernelDoc + + +class GlobSourceFiles: + """ + Parse C source code file names and directories via an Interactor. + """ + + def __init__(self, srctree=None, valid_extensions=None): + """ + Initialize valid extensions with a tuple. + + If not defined, assume default C extensions (.c and .h) + + It would be possible to use python's glob function, but it is + very slow, and it is not interactive. So, it would wait to read all + directories before actually do something. + + So, let's use our own implementation. + """ + + if not valid_extensions: + self.extensions = (".c", ".h") + else: + self.extensions = valid_extensions + + self.srctree = srctree + + def _parse_dir(self, dirname): + """Internal function to parse files recursively""" + + with os.scandir(dirname) as obj: + for entry in obj: + name = os.path.join(dirname, entry.name) + + if entry.is_dir(): + yield from self._parse_dir(name) + + if not entry.is_file(): + continue + + basename = os.path.basename(name) + + if not basename.endswith(self.extensions): + continue + + yield name + + def parse_files(self, file_list, file_not_found_cb): + """ + Define an interator to parse all source files from file_list, + handling directories if any + """ + + for fname in file_list: + if self.srctree: + f = os.path.join(self.srctree, fname) + else: + f = fname + + if os.path.isdir(f): + yield from self._parse_dir(f) + elif os.path.isfile(f): + yield f + elif file_not_found_cb: + file_not_found_cb(fname) + + +class KernelFiles(): + """ + Parse lernel-doc tags on multiple kernel source files. + """ + + def parse_file(self, fname): + """ + Parse a single Kernel source. + """ + + doc = KernelDoc(self.config, fname) + doc.run() + + return doc + + def process_export_file(self, fname): + """ + Parses EXPORT_SYMBOL* macros from a single Kernel source file. + """ + try: + with open(fname, "r", encoding="utf8", + errors="backslashreplace") as fp: + for line in fp: + KernelDoc.process_export(self.config.function_table, line) + + except IOError: + print(f"Error: Cannot open fname {fname}", fname=sys.stderr) + self.config.errors += 1 + + def file_not_found_cb(self, fname): + """ + Callback to warn if a file was not found. + """ + + self.config.log.error("Cannot find file %s", fname) + self.config.errors += 1 + + def __init__(self, files=None, verbose=False, out_style=None, + werror=False, wreturn=False, wshort_desc=False, + wcontents_before_sections=False, + logger=None, modulename=None, export_file=None): + """ + Initialize startup variables and parse all files + """ + + if not verbose: + verbose = bool(os.environ.get("KBUILD_VERBOSE", 0)) + + if not modulename: + modulename = "Kernel API" + + dt = datetime.now() + if os.environ.get("KBUILD_BUILD_TIMESTAMP", None): + # use UTC TZ + to_zone = tz.gettz('UTC') + dt = dt.astimezone(to_zone) + + if not werror: + kcflags = os.environ.get("KCFLAGS", None) + if kcflags: + match = re.search(r"(\s|^)-Werror(\s|$)/", kcflags) + if match: + werror = True + + # reading this variable is for backwards compat just in case + # someone was calling it with the variable from outside the + # kernel's build system + kdoc_werror = os.environ.get("KDOC_WERROR", None) + if kdoc_werror: + werror = kdoc_werror + + # Set global config data used on all files + self.config = argparse.Namespace + + self.config.verbose = verbose + self.config.werror = werror + self.config.wreturn = wreturn + self.config.wshort_desc = wshort_desc + self.config.wcontents_before_sections = wcontents_before_sections + self.config.modulename = modulename + + self.config.function_table = set() + self.config.source_map = {} + + if not logger: + self.config.log = logging.getLogger("kernel-doc") + else: + self.config.log = logger + + self.config.kernel_version = os.environ.get("KERNELVERSION", + "unknown kernel version'") + self.config.src_tree = os.environ.get("SRCTREE", None) + + self.out_style = out_style + self.export_file = export_file + + # Initialize internal variables + + self.config.errors = 0 + self.results = [] + + self.file_list = files + self.files = set() + + def parse(self): + """ + Parse all files + """ + + glob = GlobSourceFiles(srctree=self.config.src_tree) + + # Let's use a set here to avoid duplicating files + + for fname in glob.parse_files(self.file_list, self.file_not_found_cb): + if fname in self.files: + continue + + self.files.add(fname) + + res = self.parse_file(fname) + self.results.append((res.fname, res.entries)) + + if not self.files: + sys.exit(1) + + # If a list of export files was provided, parse EXPORT_SYMBOL* + # from the ones not already parsed + + if self.export_file: + files = self.files + + glob = GlobSourceFiles(srctree=self.config.src_tree) + + for fname in glob.parse_files(self.export_file, + self.file_not_found_cb): + if fname not in files: + files.add(fname) + + self.process_export_file(fname) + + def out_msg(self, fname, name, arg): + """ + Output messages from a file name using the output style filtering. + + If output type was not handled by the syler, return False. + """ + + # NOTE: we can add rules here to filter out unwanted parts, + # although OutputFormat.msg already does that. + + return self.out_style.msg(fname, name, arg) + + def msg(self, enable_lineno=False, export=False, internal=False, + symbol=None, nosymbol=None): + """ + Interacts over the kernel-doc results and output messages. + """ + + function_table = self.config.function_table + + if symbol: + for s in symbol: + function_table.add(s) + + # Output none mode: only warnings will be shown + if not self.out_style: + return + + self.out_style.set_config(self.config) + + self.out_style.set_filter(export, internal, symbol, nosymbol, + function_table, enable_lineno) + + for fname, arg_tuple in self.results: + for name, arg in arg_tuple: + if self.out_msg(fname, name, arg): + ln = arg.get("ln", 0) + dtype = arg.get('type', "") + + self.config.log.warning("%s:%d Can't handle %s", + fname, ln, dtype) -- 2.51.0 From 1d6fea640e6ccb2c4ee0b492270562e89ba2805f Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 8 Apr 2025 18:09:13 +0800 Subject: [PATCH 02/16] scripts/kernel-doc.py: move output classes to a separate file In preparation for letting kerneldoc Sphinx extension to import Python libraries, move kernel-doc output logic to a separate file. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/81087eff25d11c265019a8631f7fc8d3904795d0.1744106242.git.mchehab+huawei@kernel.org --- scripts/kernel-doc.py | 727 +------------------------------ scripts/lib/kdoc/kdoc_output.py | 736 ++++++++++++++++++++++++++++++++ 2 files changed, 739 insertions(+), 724 deletions(-) create mode 100755 scripts/lib/kdoc/kdoc_output.py diff --git a/scripts/kernel-doc.py b/scripts/kernel-doc.py index d09ada2d862a..abff78e9160f 100755 --- a/scripts/kernel-doc.py +++ b/scripts/kernel-doc.py @@ -2,9 +2,7 @@ # SPDX-License-Identifier: GPL-2.0 # Copyright(c) 2025: Mauro Carvalho Chehab . # -# pylint: disable=R0902,R0903,R0904,R0911,R0912,R0913,R0914,R0915,R0917,R1702 -# pylint: disable=C0302,C0103,C0301 -# pylint: disable=C0116,C0115,W0511,W0613 +# pylint: disable=C0103 # # Converted from the kernel-doc script originally written in Perl # under GPLv2, copyrighted since 1998 by the following authors: @@ -102,14 +100,8 @@ documentation comment syntax. import argparse import logging import os -import re import sys -from datetime import datetime -from pprint import pformat - -from dateutil import tz - # Import Python modules LIB_DIR = "lib/kdoc" @@ -117,721 +109,8 @@ SRC_DIR = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.join(SRC_DIR, LIB_DIR)) -from kdoc_parser import KernelDoc, type_param -from kdoc_re import Re -from kdoc_files import KernelFiles - -function_pointer = Re(r"([^\(]*\(\*)\s*\)\s*\(([^\)]*)\)", cache=False) - -# match expressions used to find embedded type information -type_constant = Re(r"\b``([^\`]+)``\b", cache=False) -type_constant2 = Re(r"\%([-_*\w]+)", cache=False) -type_func = Re(r"(\w+)\(\)", cache=False) -type_param_ref = Re(r"([\!~\*]?)\@(\w*((\.\w+)|(->\w+))*(\.\.\.)?)", cache=False) - -# Special RST handling for func ptr params -type_fp_param = Re(r"\@(\w+)\(\)", cache=False) - -# Special RST handling for structs with func ptr params -type_fp_param2 = Re(r"\@(\w+->\S+)\(\)", cache=False) - -type_env = Re(r"(\$\w+)", cache=False) -type_enum = Re(r"\&(enum\s*([_\w]+))", cache=False) -type_struct = Re(r"\&(struct\s*([_\w]+))", cache=False) -type_typedef = Re(r"\&(typedef\s*([_\w]+))", cache=False) -type_union = Re(r"\&(union\s*([_\w]+))", cache=False) -type_member = Re(r"\&([_\w]+)(\.|->)([_\w]+)", cache=False) -type_fallback = Re(r"\&([_\w]+)", cache=False) -type_member_func = type_member + Re(r"\(\)", cache=False) - - -class OutputFormat: - # output mode. - OUTPUT_ALL = 0 # output all symbols and doc sections - OUTPUT_INCLUDE = 1 # output only specified symbols - OUTPUT_EXPORTED = 2 # output exported symbols - OUTPUT_INTERNAL = 3 # output non-exported symbols - - # Virtual member to be overriden at the inherited classes - highlights = [] - - def __init__(self): - """Declare internal vars and set mode to OUTPUT_ALL""" - - self.out_mode = self.OUTPUT_ALL - self.enable_lineno = None - self.nosymbol = {} - self.symbol = None - self.function_table = set() - self.config = None - - def set_config(self, config): - self.config = config - - def set_filter(self, export, internal, symbol, nosymbol, function_table, - enable_lineno): - """ - Initialize filter variables according with the requested mode. - - Only one choice is valid between export, internal and symbol. - - The nosymbol filter can be used on all modes. - """ - - self.enable_lineno = enable_lineno - - if symbol: - self.out_mode = self.OUTPUT_INCLUDE - function_table = symbol - elif export: - self.out_mode = self.OUTPUT_EXPORTED - elif internal: - self.out_mode = self.OUTPUT_INTERNAL - else: - self.out_mode = self.OUTPUT_ALL - - if nosymbol: - self.nosymbol = set(nosymbol) - - if function_table: - self.function_table = function_table - - def highlight_block(self, block): - """ - Apply the RST highlights to a sub-block of text. - """ - - for r, sub in self.highlights: - block = r.sub(sub, block) - - return block - - def check_doc(self, name): - """Check if DOC should be output""" - - if self.out_mode == self.OUTPUT_ALL: - return True - - if self.out_mode == self.OUTPUT_INCLUDE: - if name in self.nosymbol: - return False - - if name in self.function_table: - return True - - return False - - def check_declaration(self, dtype, name): - if name in self.nosymbol: - return False - - if self.out_mode == self.OUTPUT_ALL: - return True - - if self.out_mode in [ self.OUTPUT_INCLUDE, self.OUTPUT_EXPORTED ]: - if name in self.function_table: - return True - - if self.out_mode == self.OUTPUT_INTERNAL: - if dtype != "function": - return True - - if name not in self.function_table: - return True - - return False - - def check_function(self, fname, name, args): - return True - - def check_enum(self, fname, name, args): - return True - - def check_typedef(self, fname, name, args): - return True - - def msg(self, fname, name, args): - - dtype = args.get('type', "") - - if dtype == "doc": - self.out_doc(fname, name, args) - return False - - if not self.check_declaration(dtype, name): - return False - - if dtype == "function": - self.out_function(fname, name, args) - return False - - if dtype == "enum": - self.out_enum(fname, name, args) - return False - - if dtype == "typedef": - self.out_typedef(fname, name, args) - return False - - if dtype in ["struct", "union"]: - self.out_struct(fname, name, args) - return False - - # Warn if some type requires an output logic - self.config.log.warning("doesn't now how to output '%s' block", - dtype) - - return True - - # Virtual methods to be overridden by inherited classes - def out_doc(self, fname, name, args): - pass - - def out_function(self, fname, name, args): - pass - - def out_enum(self, fname, name, args): - pass - - def out_typedef(self, fname, name, args): - pass - - def out_struct(self, fname, name, args): - pass - - -class RestFormat(OutputFormat): - # """Consts and functions used by ReST output""" - - highlights = [ - (type_constant, r"``\1``"), - (type_constant2, r"``\1``"), - - # Note: need to escape () to avoid func matching later - (type_member_func, r":c:type:`\1\2\3\\(\\) <\1>`"), - (type_member, r":c:type:`\1\2\3 <\1>`"), - (type_fp_param, r"**\1\\(\\)**"), - (type_fp_param2, r"**\1\\(\\)**"), - (type_func, r"\1()"), - (type_enum, r":c:type:`\1 <\2>`"), - (type_struct, r":c:type:`\1 <\2>`"), - (type_typedef, r":c:type:`\1 <\2>`"), - (type_union, r":c:type:`\1 <\2>`"), - - # in rst this can refer to any type - (type_fallback, r":c:type:`\1`"), - (type_param_ref, r"**\1\2**") - ] - blankline = "\n" - - sphinx_literal = Re(r'^[^.].*::$', cache=False) - sphinx_cblock = Re(r'^\.\.\ +code-block::', cache=False) - - def __init__(self): - """ - Creates class variables. - - Not really mandatory, but it is a good coding style and makes - pylint happy. - """ - - super().__init__() - self.lineprefix = "" - - def print_lineno (self, ln): - """Outputs a line number""" - - if self.enable_lineno and ln: - print(f".. LINENO {ln}") - - def output_highlight(self, args): - input_text = args - output = "" - in_literal = False - litprefix = "" - block = "" - - for line in input_text.strip("\n").split("\n"): - - # If we're in a literal block, see if we should drop out of it. - # Otherwise, pass the line straight through unmunged. - if in_literal: - if line.strip(): # If the line is not blank - # If this is the first non-blank line in a literal block, - # figure out the proper indent. - if not litprefix: - r = Re(r'^(\s*)') - if r.match(line): - litprefix = '^' + r.group(1) - else: - litprefix = "" - - output += line + "\n" - elif not Re(litprefix).match(line): - in_literal = False - else: - output += line + "\n" - else: - output += line + "\n" - - # Not in a literal block (or just dropped out) - if not in_literal: - block += line + "\n" - if self.sphinx_literal.match(line) or self.sphinx_cblock.match(line): - in_literal = True - litprefix = "" - output += self.highlight_block(block) - block = "" - - # Handle any remaining block - if block: - output += self.highlight_block(block) - - # Print the output with the line prefix - for line in output.strip("\n").split("\n"): - print(self.lineprefix + line) - - def out_section(self, args, out_reference=False): - """ - Outputs a block section. - - This could use some work; it's used to output the DOC: sections, and - starts by putting out the name of the doc section itself, but that - tends to duplicate a header already in the template file. - """ - - sectionlist = args.get('sectionlist', []) - sections = args.get('sections', {}) - section_start_lines = args.get('section_start_lines', {}) - - for section in sectionlist: - # Skip sections that are in the nosymbol_table - if section in self.nosymbol: - continue - - if not self.out_mode == self.OUTPUT_INCLUDE: - if out_reference: - print(f".. _{section}:\n") - - if not self.symbol: - print(f'{self.lineprefix}**{section}**\n') - - self.print_lineno(section_start_lines.get(section, 0)) - self.output_highlight(sections[section]) - print() - print() - - def out_doc(self, fname, name, args): - if not self.check_doc(name): - return - - self.out_section(args, out_reference=True) - - def out_function(self, fname, name, args): - - oldprefix = self.lineprefix - signature = "" - - func_macro = args.get('func_macro', False) - if func_macro: - signature = args['function'] - else: - if args.get('functiontype'): - signature = args['functiontype'] + " " - signature += args['function'] + " (" - - parameterlist = args.get('parameterlist', []) - parameterdescs = args.get('parameterdescs', {}) - parameterdesc_start_lines = args.get('parameterdesc_start_lines', {}) - - ln = args.get('ln', 0) - - count = 0 - for parameter in parameterlist: - if count != 0: - signature += ", " - count += 1 - dtype = args['parametertypes'].get(parameter, "") - - if function_pointer.search(dtype): - signature += function_pointer.group(1) + parameter + function_pointer.group(3) - else: - signature += dtype - - if not func_macro: - signature += ")" - - if args.get('typedef') or not args.get('functiontype'): - print(f".. c:macro:: {args['function']}\n") - - if args.get('typedef'): - self.print_lineno(ln) - print(" **Typedef**: ", end="") - self.lineprefix = "" - self.output_highlight(args.get('purpose', "")) - print("\n\n**Syntax**\n") - print(f" ``{signature}``\n") - else: - print(f"``{signature}``\n") - else: - print(f".. c:function:: {signature}\n") - - if not args.get('typedef'): - self.print_lineno(ln) - self.lineprefix = " " - self.output_highlight(args.get('purpose', "")) - print() - - # Put descriptive text into a container (HTML
) to help set - # function prototypes apart - self.lineprefix = " " - - if parameterlist: - print(".. container:: kernelindent\n") - print(f"{self.lineprefix}**Parameters**\n") - - for parameter in parameterlist: - parameter_name = Re(r'\[.*').sub('', parameter) - dtype = args['parametertypes'].get(parameter, "") - - if dtype: - print(f"{self.lineprefix}``{dtype}``") - else: - print(f"{self.lineprefix}``{parameter}``") - - self.print_lineno(parameterdesc_start_lines.get(parameter_name, 0)) - - self.lineprefix = " " - if parameter_name in parameterdescs and \ - parameterdescs[parameter_name] != KernelDoc.undescribed: - - self.output_highlight(parameterdescs[parameter_name]) - print() - else: - print(f"{self.lineprefix}*undescribed*\n") - self.lineprefix = " " - - self.out_section(args) - self.lineprefix = oldprefix - - def out_enum(self, fname, name, args): - - oldprefix = self.lineprefix - name = args.get('enum', '') - parameterlist = args.get('parameterlist', []) - parameterdescs = args.get('parameterdescs', {}) - ln = args.get('ln', 0) - - print(f"\n\n.. c:enum:: {name}\n") - - self.print_lineno(ln) - self.lineprefix = " " - self.output_highlight(args.get('purpose', '')) - print() - - print(".. container:: kernelindent\n") - outer = self.lineprefix + " " - self.lineprefix = outer + " " - print(f"{outer}**Constants**\n") - - for parameter in parameterlist: - print(f"{outer}``{parameter}``") - - if parameterdescs.get(parameter, '') != KernelDoc.undescribed: - self.output_highlight(parameterdescs[parameter]) - else: - print(f"{self.lineprefix}*undescribed*\n") - print() - - self.lineprefix = oldprefix - self.out_section(args) - - def out_typedef(self, fname, name, args): - - oldprefix = self.lineprefix - name = args.get('typedef', '') - ln = args.get('ln', 0) - - print(f"\n\n.. c:type:: {name}\n") - - self.print_lineno(ln) - self.lineprefix = " " - - self.output_highlight(args.get('purpose', '')) - - print() - - self.lineprefix = oldprefix - self.out_section(args) - - def out_struct(self, fname, name, args): - - name = args.get('struct', "") - purpose = args.get('purpose', "") - declaration = args.get('definition', "") - dtype = args.get('type', "struct") - ln = args.get('ln', 0) - - parameterlist = args.get('parameterlist', []) - parameterdescs = args.get('parameterdescs', {}) - parameterdesc_start_lines = args.get('parameterdesc_start_lines', {}) - - print(f"\n\n.. c:{dtype}:: {name}\n") - - self.print_lineno(ln) - - oldprefix = self.lineprefix - self.lineprefix += " " - - self.output_highlight(purpose) - print() - - print(".. container:: kernelindent\n") - print(f"{self.lineprefix}**Definition**::\n") - - self.lineprefix = self.lineprefix + " " - - declaration = declaration.replace("\t", self.lineprefix) - - print(f"{self.lineprefix}{dtype} {name}" + ' {') - print(f"{declaration}{self.lineprefix}" + "};\n") - - self.lineprefix = " " - print(f"{self.lineprefix}**Members**\n") - for parameter in parameterlist: - if not parameter or parameter.startswith("#"): - continue - - parameter_name = parameter.split("[", maxsplit=1)[0] - - if parameterdescs.get(parameter_name) == KernelDoc.undescribed: - continue - - self.print_lineno(parameterdesc_start_lines.get(parameter_name, 0)) - - print(f"{self.lineprefix}``{parameter}``") - - self.lineprefix = " " - self.output_highlight(parameterdescs[parameter_name]) - self.lineprefix = " " - - print() - - print() - - self.lineprefix = oldprefix - self.out_section(args) - - -class ManFormat(OutputFormat): - """Consts and functions used by man pages output""" - - highlights = ( - (type_constant, r"\1"), - (type_constant2, r"\1"), - (type_func, r"\\fB\1\\fP"), - (type_enum, r"\\fI\1\\fP"), - (type_struct, r"\\fI\1\\fP"), - (type_typedef, r"\\fI\1\\fP"), - (type_union, r"\\fI\1\\fP"), - (type_param, r"\\fI\1\\fP"), - (type_param_ref, r"\\fI\1\2\\fP"), - (type_member, r"\\fI\1\2\3\\fP"), - (type_fallback, r"\\fI\1\\fP") - ) - blankline = "" - - def __init__(self): - """ - Creates class variables. - - Not really mandatory, but it is a good coding style and makes - pylint happy. - """ - - super().__init__() - - dt = datetime.now() - if os.environ.get("KBUILD_BUILD_TIMESTAMP", None): - # use UTC TZ - to_zone = tz.gettz('UTC') - dt = dt.astimezone(to_zone) - - self.man_date = dt.strftime("%B %Y") - - def output_highlight(self, block): - - contents = self.highlight_block(block) - - if isinstance(contents, list): - contents = "\n".join(contents) - - for line in contents.strip("\n").split("\n"): - line = Re(r"^\s*").sub("", line) - - if line and line[0] == ".": - print("\\&" + line) - else: - print(line) - - def out_doc(self, fname, name, args): - module = args.get('module') - sectionlist = args.get('sectionlist', []) - sections = args.get('sections', {}) - - print(f'.TH "{module}" 9 "{module}" "{self.man_date}" "API Manual" LINUX') - - for section in sectionlist: - print(f'.SH "{section}"') - self.output_highlight(sections.get(section)) - - def out_function(self, fname, name, args): - """output function in man""" - - parameterlist = args.get('parameterlist', []) - parameterdescs = args.get('parameterdescs', {}) - sectionlist = args.get('sectionlist', []) - sections = args.get('sections', {}) - - print(f'.TH "{args['function']}" 9 "{args['function']}" "{self.man_date}" "Kernel Hacker\'s Manual" LINUX') - - print(".SH NAME") - print(f"{args['function']} \\- {args['purpose']}") - - print(".SH SYNOPSIS") - if args.get('functiontype', ''): - print(f'.B "{args['functiontype']}" {args['function']}') - else: - print(f'.B "{args['function']}') - - count = 0 - parenth = "(" - post = "," - - for parameter in parameterlist: - if count == len(parameterlist) - 1: - post = ");" - - dtype = args['parametertypes'].get(parameter, "") - if function_pointer.match(dtype): - # Pointer-to-function - print(f'".BI "{parenth}{function_pointer.group(1)}" " ") ({function_pointer.group(2)}){post}"') - else: - dtype = Re(r'([^\*])$').sub(r'\1 ', dtype) - - print(f'.BI "{parenth}{dtype}" "{post}"') - count += 1 - parenth = "" - - if parameterlist: - print(".SH ARGUMENTS") - - for parameter in parameterlist: - parameter_name = re.sub(r'\[.*', '', parameter) - - print(f'.IP "{parameter}" 12') - self.output_highlight(parameterdescs.get(parameter_name, "")) - - for section in sectionlist: - print(f'.SH "{section.upper()}"') - self.output_highlight(sections[section]) - - def out_enum(self, fname, name, args): - - name = args.get('enum', '') - parameterlist = args.get('parameterlist', []) - sectionlist = args.get('sectionlist', []) - sections = args.get('sections', {}) - - print(f'.TH "{args['module']}" 9 "enum {args['enum']}" "{self.man_date}" "API Manual" LINUX') - - print(".SH NAME") - print(f"enum {args['enum']} \\- {args['purpose']}") - - print(".SH SYNOPSIS") - print(f"enum {args['enum']}" + " {") - - count = 0 - for parameter in parameterlist: - print(f'.br\n.BI " {parameter}"') - if count == len(parameterlist) - 1: - print("\n};") - else: - print(", \n.br") - - count += 1 - - print(".SH Constants") - - for parameter in parameterlist: - parameter_name = Re(r'\[.*').sub('', parameter) - print(f'.IP "{parameter}" 12') - self.output_highlight(args['parameterdescs'].get(parameter_name, "")) - - for section in sectionlist: - print(f'.SH "{section}"') - self.output_highlight(sections[section]) - - def out_typedef(self, fname, name, args): - module = args.get('module') - typedef = args.get('typedef') - purpose = args.get('purpose') - sectionlist = args.get('sectionlist', []) - sections = args.get('sections', {}) - - print(f'.TH "{module}" 9 "{typedef}" "{self.man_date}" "API Manual" LINUX') - - print(".SH NAME") - print(f"typedef {typedef} \\- {purpose}") - - for section in sectionlist: - print(f'.SH "{section}"') - self.output_highlight(sections.get(section)) - - def out_struct(self, fname, name, args): - module = args.get('module') - struct_type = args.get('type') - struct_name = args.get('struct') - purpose = args.get('purpose') - definition = args.get('definition') - sectionlist = args.get('sectionlist', []) - parameterlist = args.get('parameterlist', []) - sections = args.get('sections', {}) - parameterdescs = args.get('parameterdescs', {}) - - print(f'.TH "{module}" 9 "{struct_type} {struct_name}" "{self.man_date}" "API Manual" LINUX') - - print(".SH NAME") - print(f"{struct_type} {struct_name} \\- {purpose}") - - # Replace tabs with two spaces and handle newlines - declaration = definition.replace("\t", " ") - declaration = Re(r"\n").sub('"\n.br\n.BI "', declaration) - - print(".SH SYNOPSIS") - print(f"{struct_type} {struct_name} " + "{" +"\n.br") - print(f'.BI "{declaration}\n' + "};\n.br\n") - - print(".SH Members") - for parameter in parameterlist: - if parameter.startswith("#"): - continue - - parameter_name = re.sub(r"\[.*", "", parameter) - - if parameterdescs.get(parameter_name) == KernelDoc.undescribed: - continue - - print(f'.IP "{parameter}" 12') - self.output_highlight(parameterdescs.get(parameter_name)) - - for section in sectionlist: - print(f'.SH "{section}"') - self.output_highlight(sections.get(section)) - - -# Command line interface - +from kdoc_files import KernelFiles # pylint: disable=C0413 +from kdoc_output import RestFormat, ManFormat # pylint: disable=C0413 DESC = """ Read C language source or header FILEs, extract embedded documentation comments, diff --git a/scripts/lib/kdoc/kdoc_output.py b/scripts/lib/kdoc/kdoc_output.py new file mode 100755 index 000000000000..24e40b3e7d1d --- /dev/null +++ b/scripts/lib/kdoc/kdoc_output.py @@ -0,0 +1,736 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +# Copyright(c) 2025: Mauro Carvalho Chehab . +# +# pylint: disable=C0301,R0911,R0912,R0913,R0914,R0915,R0917 + +# TODO: implement warning filtering + +""" +Implement output filters to print kernel-doc documentation. + +The implementation uses a virtual base class (OutputFormat) which +contains a dispatches to virtual methods, and some code to filter +out output messages. + +The actual implementation is done on one separate class per each type +of output. Currently, there are output classes for ReST and man/troff. +""" + +import os +import re +from datetime import datetime + +from dateutil import tz + +from kdoc_parser import KernelDoc, type_param +from kdoc_re import Re + + +function_pointer = Re(r"([^\(]*\(\*)\s*\)\s*\(([^\)]*)\)", cache=False) + +# match expressions used to find embedded type information +type_constant = Re(r"\b``([^\`]+)``\b", cache=False) +type_constant2 = Re(r"\%([-_*\w]+)", cache=False) +type_func = Re(r"(\w+)\(\)", cache=False) +type_param_ref = Re(r"([\!~\*]?)\@(\w*((\.\w+)|(->\w+))*(\.\.\.)?)", cache=False) + +# Special RST handling for func ptr params +type_fp_param = Re(r"\@(\w+)\(\)", cache=False) + +# Special RST handling for structs with func ptr params +type_fp_param2 = Re(r"\@(\w+->\S+)\(\)", cache=False) + +type_env = Re(r"(\$\w+)", cache=False) +type_enum = Re(r"\&(enum\s*([_\w]+))", cache=False) +type_struct = Re(r"\&(struct\s*([_\w]+))", cache=False) +type_typedef = Re(r"\&(typedef\s*([_\w]+))", cache=False) +type_union = Re(r"\&(union\s*([_\w]+))", cache=False) +type_member = Re(r"\&([_\w]+)(\.|->)([_\w]+)", cache=False) +type_fallback = Re(r"\&([_\w]+)", cache=False) +type_member_func = type_member + Re(r"\(\)", cache=False) + + +class OutputFormat: + # output mode. + OUTPUT_ALL = 0 # output all symbols and doc sections + OUTPUT_INCLUDE = 1 # output only specified symbols + OUTPUT_EXPORTED = 2 # output exported symbols + OUTPUT_INTERNAL = 3 # output non-exported symbols + + # Virtual member to be overriden at the inherited classes + highlights = [] + + def __init__(self): + """Declare internal vars and set mode to OUTPUT_ALL""" + + self.out_mode = self.OUTPUT_ALL + self.enable_lineno = None + self.nosymbol = {} + self.symbol = None + self.function_table = set() + self.config = None + + def set_config(self, config): + self.config = config + + def set_filter(self, export, internal, symbol, nosymbol, function_table, + enable_lineno): + """ + Initialize filter variables according with the requested mode. + + Only one choice is valid between export, internal and symbol. + + The nosymbol filter can be used on all modes. + """ + + self.enable_lineno = enable_lineno + + if symbol: + self.out_mode = self.OUTPUT_INCLUDE + function_table = symbol + elif export: + self.out_mode = self.OUTPUT_EXPORTED + elif internal: + self.out_mode = self.OUTPUT_INTERNAL + else: + self.out_mode = self.OUTPUT_ALL + + if nosymbol: + self.nosymbol = set(nosymbol) + + if function_table: + self.function_table = function_table + + def highlight_block(self, block): + """ + Apply the RST highlights to a sub-block of text. + """ + + for r, sub in self.highlights: + block = r.sub(sub, block) + + return block + + def check_doc(self, name): + """Check if DOC should be output""" + + if self.out_mode == self.OUTPUT_ALL: + return True + + if self.out_mode == self.OUTPUT_INCLUDE: + if name in self.nosymbol: + return False + + if name in self.function_table: + return True + + return False + + def check_declaration(self, dtype, name): + if name in self.nosymbol: + return False + + if self.out_mode == self.OUTPUT_ALL: + return True + + if self.out_mode in [self.OUTPUT_INCLUDE, self.OUTPUT_EXPORTED]: + if name in self.function_table: + return True + + if self.out_mode == self.OUTPUT_INTERNAL: + if dtype != "function": + return True + + if name not in self.function_table: + return True + + return False + + def check_function(self, fname, name, args): + return True + + def check_enum(self, fname, name, args): + return True + + def check_typedef(self, fname, name, args): + return True + + def msg(self, fname, name, args): + + dtype = args.get('type', "") + + if dtype == "doc": + self.out_doc(fname, name, args) + return False + + if not self.check_declaration(dtype, name): + return False + + if dtype == "function": + self.out_function(fname, name, args) + return False + + if dtype == "enum": + self.out_enum(fname, name, args) + return False + + if dtype == "typedef": + self.out_typedef(fname, name, args) + return False + + if dtype in ["struct", "union"]: + self.out_struct(fname, name, args) + return False + + # Warn if some type requires an output logic + self.config.log.warning("doesn't now how to output '%s' block", + dtype) + + return True + + # Virtual methods to be overridden by inherited classes + def out_doc(self, fname, name, args): + pass + + def out_function(self, fname, name, args): + pass + + def out_enum(self, fname, name, args): + pass + + def out_typedef(self, fname, name, args): + pass + + def out_struct(self, fname, name, args): + pass + + +class RestFormat(OutputFormat): + # """Consts and functions used by ReST output""" + + highlights = [ + (type_constant, r"``\1``"), + (type_constant2, r"``\1``"), + + # Note: need to escape () to avoid func matching later + (type_member_func, r":c:type:`\1\2\3\\(\\) <\1>`"), + (type_member, r":c:type:`\1\2\3 <\1>`"), + (type_fp_param, r"**\1\\(\\)**"), + (type_fp_param2, r"**\1\\(\\)**"), + (type_func, r"\1()"), + (type_enum, r":c:type:`\1 <\2>`"), + (type_struct, r":c:type:`\1 <\2>`"), + (type_typedef, r":c:type:`\1 <\2>`"), + (type_union, r":c:type:`\1 <\2>`"), + + # in rst this can refer to any type + (type_fallback, r":c:type:`\1`"), + (type_param_ref, r"**\1\2**") + ] + blankline = "\n" + + sphinx_literal = Re(r'^[^.].*::$', cache=False) + sphinx_cblock = Re(r'^\.\.\ +code-block::', cache=False) + + def __init__(self): + """ + Creates class variables. + + Not really mandatory, but it is a good coding style and makes + pylint happy. + """ + + super().__init__() + self.lineprefix = "" + + def print_lineno(self, ln): + """Outputs a line number""" + + if self.enable_lineno and ln: + print(f".. LINENO {ln}") + + def output_highlight(self, args): + input_text = args + output = "" + in_literal = False + litprefix = "" + block = "" + + for line in input_text.strip("\n").split("\n"): + + # If we're in a literal block, see if we should drop out of it. + # Otherwise, pass the line straight through unmunged. + if in_literal: + if line.strip(): # If the line is not blank + # If this is the first non-blank line in a literal block, + # figure out the proper indent. + if not litprefix: + r = Re(r'^(\s*)') + if r.match(line): + litprefix = '^' + r.group(1) + else: + litprefix = "" + + output += line + "\n" + elif not Re(litprefix).match(line): + in_literal = False + else: + output += line + "\n" + else: + output += line + "\n" + + # Not in a literal block (or just dropped out) + if not in_literal: + block += line + "\n" + if self.sphinx_literal.match(line) or self.sphinx_cblock.match(line): + in_literal = True + litprefix = "" + output += self.highlight_block(block) + block = "" + + # Handle any remaining block + if block: + output += self.highlight_block(block) + + # Print the output with the line prefix + for line in output.strip("\n").split("\n"): + print(self.lineprefix + line) + + def out_section(self, args, out_reference=False): + """ + Outputs a block section. + + This could use some work; it's used to output the DOC: sections, and + starts by putting out the name of the doc section itself, but that + tends to duplicate a header already in the template file. + """ + + sectionlist = args.get('sectionlist', []) + sections = args.get('sections', {}) + section_start_lines = args.get('section_start_lines', {}) + + for section in sectionlist: + # Skip sections that are in the nosymbol_table + if section in self.nosymbol: + continue + + if not self.out_mode == self.OUTPUT_INCLUDE: + if out_reference: + print(f".. _{section}:\n") + + if not self.symbol: + print(f'{self.lineprefix}**{section}**\n') + + self.print_lineno(section_start_lines.get(section, 0)) + self.output_highlight(sections[section]) + print() + print() + + def out_doc(self, fname, name, args): + if not self.check_doc(name): + return + + self.out_section(args, out_reference=True) + + def out_function(self, fname, name, args): + + oldprefix = self.lineprefix + signature = "" + + func_macro = args.get('func_macro', False) + if func_macro: + signature = args['function'] + else: + if args.get('functiontype'): + signature = args['functiontype'] + " " + signature += args['function'] + " (" + + parameterlist = args.get('parameterlist', []) + parameterdescs = args.get('parameterdescs', {}) + parameterdesc_start_lines = args.get('parameterdesc_start_lines', {}) + + ln = args.get('ln', 0) + + count = 0 + for parameter in parameterlist: + if count != 0: + signature += ", " + count += 1 + dtype = args['parametertypes'].get(parameter, "") + + if function_pointer.search(dtype): + signature += function_pointer.group(1) + parameter + function_pointer.group(3) + else: + signature += dtype + + if not func_macro: + signature += ")" + + if args.get('typedef') or not args.get('functiontype'): + print(f".. c:macro:: {args['function']}\n") + + if args.get('typedef'): + self.print_lineno(ln) + print(" **Typedef**: ", end="") + self.lineprefix = "" + self.output_highlight(args.get('purpose', "")) + print("\n\n**Syntax**\n") + print(f" ``{signature}``\n") + else: + print(f"``{signature}``\n") + else: + print(f".. c:function:: {signature}\n") + + if not args.get('typedef'): + self.print_lineno(ln) + self.lineprefix = " " + self.output_highlight(args.get('purpose', "")) + print() + + # Put descriptive text into a container (HTML
) to help set + # function prototypes apart + self.lineprefix = " " + + if parameterlist: + print(".. container:: kernelindent\n") + print(f"{self.lineprefix}**Parameters**\n") + + for parameter in parameterlist: + parameter_name = Re(r'\[.*').sub('', parameter) + dtype = args['parametertypes'].get(parameter, "") + + if dtype: + print(f"{self.lineprefix}``{dtype}``") + else: + print(f"{self.lineprefix}``{parameter}``") + + self.print_lineno(parameterdesc_start_lines.get(parameter_name, 0)) + + self.lineprefix = " " + if parameter_name in parameterdescs and \ + parameterdescs[parameter_name] != KernelDoc.undescribed: + + self.output_highlight(parameterdescs[parameter_name]) + print() + else: + print(f"{self.lineprefix}*undescribed*\n") + self.lineprefix = " " + + self.out_section(args) + self.lineprefix = oldprefix + + def out_enum(self, fname, name, args): + + oldprefix = self.lineprefix + name = args.get('enum', '') + parameterlist = args.get('parameterlist', []) + parameterdescs = args.get('parameterdescs', {}) + ln = args.get('ln', 0) + + print(f"\n\n.. c:enum:: {name}\n") + + self.print_lineno(ln) + self.lineprefix = " " + self.output_highlight(args.get('purpose', '')) + print() + + print(".. container:: kernelindent\n") + outer = self.lineprefix + " " + self.lineprefix = outer + " " + print(f"{outer}**Constants**\n") + + for parameter in parameterlist: + print(f"{outer}``{parameter}``") + + if parameterdescs.get(parameter, '') != KernelDoc.undescribed: + self.output_highlight(parameterdescs[parameter]) + else: + print(f"{self.lineprefix}*undescribed*\n") + print() + + self.lineprefix = oldprefix + self.out_section(args) + + def out_typedef(self, fname, name, args): + + oldprefix = self.lineprefix + name = args.get('typedef', '') + ln = args.get('ln', 0) + + print(f"\n\n.. c:type:: {name}\n") + + self.print_lineno(ln) + self.lineprefix = " " + + self.output_highlight(args.get('purpose', '')) + + print() + + self.lineprefix = oldprefix + self.out_section(args) + + def out_struct(self, fname, name, args): + + name = args.get('struct', "") + purpose = args.get('purpose', "") + declaration = args.get('definition', "") + dtype = args.get('type', "struct") + ln = args.get('ln', 0) + + parameterlist = args.get('parameterlist', []) + parameterdescs = args.get('parameterdescs', {}) + parameterdesc_start_lines = args.get('parameterdesc_start_lines', {}) + + print(f"\n\n.. c:{dtype}:: {name}\n") + + self.print_lineno(ln) + + oldprefix = self.lineprefix + self.lineprefix += " " + + self.output_highlight(purpose) + print() + + print(".. container:: kernelindent\n") + print(f"{self.lineprefix}**Definition**::\n") + + self.lineprefix = self.lineprefix + " " + + declaration = declaration.replace("\t", self.lineprefix) + + print(f"{self.lineprefix}{dtype} {name}" + ' {') + print(f"{declaration}{self.lineprefix}" + "};\n") + + self.lineprefix = " " + print(f"{self.lineprefix}**Members**\n") + for parameter in parameterlist: + if not parameter or parameter.startswith("#"): + continue + + parameter_name = parameter.split("[", maxsplit=1)[0] + + if parameterdescs.get(parameter_name) == KernelDoc.undescribed: + continue + + self.print_lineno(parameterdesc_start_lines.get(parameter_name, 0)) + + print(f"{self.lineprefix}``{parameter}``") + + self.lineprefix = " " + self.output_highlight(parameterdescs[parameter_name]) + self.lineprefix = " " + + print() + + print() + + self.lineprefix = oldprefix + self.out_section(args) + + +class ManFormat(OutputFormat): + """Consts and functions used by man pages output""" + + highlights = ( + (type_constant, r"\1"), + (type_constant2, r"\1"), + (type_func, r"\\fB\1\\fP"), + (type_enum, r"\\fI\1\\fP"), + (type_struct, r"\\fI\1\\fP"), + (type_typedef, r"\\fI\1\\fP"), + (type_union, r"\\fI\1\\fP"), + (type_param, r"\\fI\1\\fP"), + (type_param_ref, r"\\fI\1\2\\fP"), + (type_member, r"\\fI\1\2\3\\fP"), + (type_fallback, r"\\fI\1\\fP") + ) + blankline = "" + + def __init__(self): + """ + Creates class variables. + + Not really mandatory, but it is a good coding style and makes + pylint happy. + """ + + super().__init__() + + dt = datetime.now() + if os.environ.get("KBUILD_BUILD_TIMESTAMP", None): + # use UTC TZ + to_zone = tz.gettz('UTC') + dt = dt.astimezone(to_zone) + + self.man_date = dt.strftime("%B %Y") + + def output_highlight(self, block): + + contents = self.highlight_block(block) + + if isinstance(contents, list): + contents = "\n".join(contents) + + for line in contents.strip("\n").split("\n"): + line = Re(r"^\s*").sub("", line) + + if line and line[0] == ".": + print("\\&" + line) + else: + print(line) + + def out_doc(self, fname, name, args): + module = args.get('module') + sectionlist = args.get('sectionlist', []) + sections = args.get('sections', {}) + + print(f'.TH "{module}" 9 "{module}" "{self.man_date}" "API Manual" LINUX') + + for section in sectionlist: + print(f'.SH "{section}"') + self.output_highlight(sections.get(section)) + + def out_function(self, fname, name, args): + """output function in man""" + + parameterlist = args.get('parameterlist', []) + parameterdescs = args.get('parameterdescs', {}) + sectionlist = args.get('sectionlist', []) + sections = args.get('sections', {}) + + print(f'.TH "{args['function']}" 9 "{args['function']}" "{self.man_date}" "Kernel Hacker\'s Manual" LINUX') + + print(".SH NAME") + print(f"{args['function']} \\- {args['purpose']}") + + print(".SH SYNOPSIS") + if args.get('functiontype', ''): + print(f'.B "{args['functiontype']}" {args['function']}') + else: + print(f'.B "{args['function']}') + + count = 0 + parenth = "(" + post = "," + + for parameter in parameterlist: + if count == len(parameterlist) - 1: + post = ");" + + dtype = args['parametertypes'].get(parameter, "") + if function_pointer.match(dtype): + # Pointer-to-function + print(f'".BI "{parenth}{function_pointer.group(1)}" " ") ({function_pointer.group(2)}){post}"') + else: + dtype = Re(r'([^\*])$').sub(r'\1 ', dtype) + + print(f'.BI "{parenth}{dtype}" "{post}"') + count += 1 + parenth = "" + + if parameterlist: + print(".SH ARGUMENTS") + + for parameter in parameterlist: + parameter_name = re.sub(r'\[.*', '', parameter) + + print(f'.IP "{parameter}" 12') + self.output_highlight(parameterdescs.get(parameter_name, "")) + + for section in sectionlist: + print(f'.SH "{section.upper()}"') + self.output_highlight(sections[section]) + + def out_enum(self, fname, name, args): + + name = args.get('enum', '') + parameterlist = args.get('parameterlist', []) + sectionlist = args.get('sectionlist', []) + sections = args.get('sections', {}) + + print(f'.TH "{args['module']}" 9 "enum {args['enum']}" "{self.man_date}" "API Manual" LINUX') + + print(".SH NAME") + print(f"enum {args['enum']} \\- {args['purpose']}") + + print(".SH SYNOPSIS") + print(f"enum {args['enum']}" + " {") + + count = 0 + for parameter in parameterlist: + print(f'.br\n.BI " {parameter}"') + if count == len(parameterlist) - 1: + print("\n};") + else: + print(", \n.br") + + count += 1 + + print(".SH Constants") + + for parameter in parameterlist: + parameter_name = Re(r'\[.*').sub('', parameter) + print(f'.IP "{parameter}" 12') + self.output_highlight(args['parameterdescs'].get(parameter_name, "")) + + for section in sectionlist: + print(f'.SH "{section}"') + self.output_highlight(sections[section]) + + def out_typedef(self, fname, name, args): + module = args.get('module') + typedef = args.get('typedef') + purpose = args.get('purpose') + sectionlist = args.get('sectionlist', []) + sections = args.get('sections', {}) + + print(f'.TH "{module}" 9 "{typedef}" "{self.man_date}" "API Manual" LINUX') + + print(".SH NAME") + print(f"typedef {typedef} \\- {purpose}") + + for section in sectionlist: + print(f'.SH "{section}"') + self.output_highlight(sections.get(section)) + + def out_struct(self, fname, name, args): + module = args.get('module') + struct_type = args.get('type') + struct_name = args.get('struct') + purpose = args.get('purpose') + definition = args.get('definition') + sectionlist = args.get('sectionlist', []) + parameterlist = args.get('parameterlist', []) + sections = args.get('sections', {}) + parameterdescs = args.get('parameterdescs', {}) + + print(f'.TH "{module}" 9 "{struct_type} {struct_name}" "{self.man_date}" "API Manual" LINUX') + + print(".SH NAME") + print(f"{struct_type} {struct_name} \\- {purpose}") + + # Replace tabs with two spaces and handle newlines + declaration = definition.replace("\t", " ") + declaration = Re(r"\n").sub('"\n.br\n.BI "', declaration) + + print(".SH SYNOPSIS") + print(f"{struct_type} {struct_name} " + "{" + "\n.br") + print(f'.BI "{declaration}\n' + "};\n.br\n") + + print(".SH Members") + for parameter in parameterlist: + if parameter.startswith("#"): + continue + + parameter_name = re.sub(r"\[.*", "", parameter) + + if parameterdescs.get(parameter_name) == KernelDoc.undescribed: + continue + + print(f'.IP "{parameter}" 12') + self.output_highlight(parameterdescs.get(parameter_name)) + + for section in sectionlist: + print(f'.SH "{section}"') + self.output_highlight(sections.get(section)) -- 2.51.0 From 4fa5e411379af1baabdff088196da977799fd46e Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 8 Apr 2025 18:09:14 +0800 Subject: [PATCH 03/16] scripts/kernel-doc.py: convert message output to an interactor Instead of directly printing output messages, change kdoc classes to return an interactor with the output message, letting the actual display to happen at the command-line command. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/557304c8458f1fb4aa2e833f4bdaff953094ddcb.1744106242.git.mchehab+huawei@kernel.org --- scripts/kernel-doc.py | 9 +- scripts/lib/kdoc/kdoc_files.py | 15 ++- scripts/lib/kdoc/kdoc_output.py | 171 ++++++++++++++++---------------- 3 files changed, 104 insertions(+), 91 deletions(-) diff --git a/scripts/kernel-doc.py b/scripts/kernel-doc.py index abff78e9160f..63efec4b3f4b 100755 --- a/scripts/kernel-doc.py +++ b/scripts/kernel-doc.py @@ -283,9 +283,12 @@ def main(): kfiles.parse() - kfiles.msg(enable_lineno=args.enable_lineno, export=args.export, - internal=args.internal, symbol=args.symbol, - nosymbol=args.nosymbol) + for t in kfiles.msg(enable_lineno=args.enable_lineno, export=args.export, + internal=args.internal, symbol=args.symbol, + nosymbol=args.nosymbol): + msg = t[1] + if msg: + print(msg) # Call main method diff --git a/scripts/lib/kdoc/kdoc_files.py b/scripts/lib/kdoc/kdoc_files.py index 8bcdc7ead984..817ed98b2727 100755 --- a/scripts/lib/kdoc/kdoc_files.py +++ b/scripts/lib/kdoc/kdoc_files.py @@ -229,9 +229,10 @@ class KernelFiles(): def out_msg(self, fname, name, arg): """ - Output messages from a file name using the output style filtering. + Return output messages from a file name using the output style + filtering. - If output type was not handled by the syler, return False. + If output type was not handled by the syler, return None. """ # NOTE: we can add rules here to filter out unwanted parts, @@ -242,7 +243,8 @@ class KernelFiles(): def msg(self, enable_lineno=False, export=False, internal=False, symbol=None, nosymbol=None): """ - Interacts over the kernel-doc results and output messages. + Interacts over the kernel-doc results and output messages, + returning kernel-doc markups on each interaction """ function_table = self.config.function_table @@ -261,10 +263,15 @@ class KernelFiles(): function_table, enable_lineno) for fname, arg_tuple in self.results: + msg = "" for name, arg in arg_tuple: - if self.out_msg(fname, name, arg): + msg += self.out_msg(fname, name, arg) + + if msg is None: ln = arg.get("ln", 0) dtype = arg.get('type', "") self.config.log.warning("%s:%d Can't handle %s", fname, ln, dtype) + if msg: + yield fname, msg diff --git a/scripts/lib/kdoc/kdoc_output.py b/scripts/lib/kdoc/kdoc_output.py index 24e40b3e7d1d..fda07049ecf7 100755 --- a/scripts/lib/kdoc/kdoc_output.py +++ b/scripts/lib/kdoc/kdoc_output.py @@ -71,6 +71,8 @@ class OutputFormat: self.function_table = set() self.config = None + self.data = "" + def set_config(self, config): self.config = config @@ -157,37 +159,38 @@ class OutputFormat: return True def msg(self, fname, name, args): + self.data = "" dtype = args.get('type', "") if dtype == "doc": self.out_doc(fname, name, args) - return False + return self.data if not self.check_declaration(dtype, name): - return False + return self.data if dtype == "function": self.out_function(fname, name, args) - return False + return self.data if dtype == "enum": self.out_enum(fname, name, args) - return False + return self.data if dtype == "typedef": self.out_typedef(fname, name, args) - return False + return self.data if dtype in ["struct", "union"]: self.out_struct(fname, name, args) - return False + return self.data # Warn if some type requires an output logic self.config.log.warning("doesn't now how to output '%s' block", dtype) - return True + return None # Virtual methods to be overridden by inherited classes def out_doc(self, fname, name, args): @@ -248,7 +251,7 @@ class RestFormat(OutputFormat): """Outputs a line number""" if self.enable_lineno and ln: - print(f".. LINENO {ln}") + self.data += f".. LINENO {ln}\n" def output_highlight(self, args): input_text = args @@ -295,7 +298,7 @@ class RestFormat(OutputFormat): # Print the output with the line prefix for line in output.strip("\n").split("\n"): - print(self.lineprefix + line) + self.data += self.lineprefix + line + "\n" def out_section(self, args, out_reference=False): """ @@ -317,15 +320,15 @@ class RestFormat(OutputFormat): if not self.out_mode == self.OUTPUT_INCLUDE: if out_reference: - print(f".. _{section}:\n") + self.data += f".. _{section}:\n\n" if not self.symbol: - print(f'{self.lineprefix}**{section}**\n') + self.data += f'{self.lineprefix}**{section}**\n\n' self.print_lineno(section_start_lines.get(section, 0)) self.output_highlight(sections[section]) - print() - print() + self.data += "\n" + self.data += "\n" def out_doc(self, fname, name, args): if not self.check_doc(name): @@ -368,42 +371,42 @@ class RestFormat(OutputFormat): signature += ")" if args.get('typedef') or not args.get('functiontype'): - print(f".. c:macro:: {args['function']}\n") + self.data += f".. c:macro:: {args['function']}\n\n" if args.get('typedef'): self.print_lineno(ln) - print(" **Typedef**: ", end="") + self.data += " **Typedef**: " self.lineprefix = "" self.output_highlight(args.get('purpose', "")) - print("\n\n**Syntax**\n") - print(f" ``{signature}``\n") + self.data += "\n\n**Syntax**\n\n" + self.data += f" ``{signature}``\n\n" else: - print(f"``{signature}``\n") + self.data += f"``{signature}``\n\n" else: - print(f".. c:function:: {signature}\n") + self.data += f".. c:function:: {signature}\n\n" if not args.get('typedef'): self.print_lineno(ln) self.lineprefix = " " self.output_highlight(args.get('purpose', "")) - print() + self.data += "\n" # Put descriptive text into a container (HTML
) to help set # function prototypes apart self.lineprefix = " " if parameterlist: - print(".. container:: kernelindent\n") - print(f"{self.lineprefix}**Parameters**\n") + self.data += ".. container:: kernelindent\n\n" + self.data += f"{self.lineprefix}**Parameters**\n\n" for parameter in parameterlist: parameter_name = Re(r'\[.*').sub('', parameter) dtype = args['parametertypes'].get(parameter, "") if dtype: - print(f"{self.lineprefix}``{dtype}``") + self.data += f"{self.lineprefix}``{dtype}``\n" else: - print(f"{self.lineprefix}``{parameter}``") + self.data += f"{self.lineprefix}``{parameter}``\n" self.print_lineno(parameterdesc_start_lines.get(parameter_name, 0)) @@ -412,9 +415,9 @@ class RestFormat(OutputFormat): parameterdescs[parameter_name] != KernelDoc.undescribed: self.output_highlight(parameterdescs[parameter_name]) - print() + self.data += "\n" else: - print(f"{self.lineprefix}*undescribed*\n") + self.data += f"{self.lineprefix}*undescribed*\n\n" self.lineprefix = " " self.out_section(args) @@ -428,26 +431,26 @@ class RestFormat(OutputFormat): parameterdescs = args.get('parameterdescs', {}) ln = args.get('ln', 0) - print(f"\n\n.. c:enum:: {name}\n") + self.data += f"\n\n.. c:enum:: {name}\n\n" self.print_lineno(ln) self.lineprefix = " " self.output_highlight(args.get('purpose', '')) - print() + self.data += "\n" - print(".. container:: kernelindent\n") + self.data += ".. container:: kernelindent\n\n" outer = self.lineprefix + " " self.lineprefix = outer + " " - print(f"{outer}**Constants**\n") + self.data += f"{outer}**Constants**\n\n" for parameter in parameterlist: - print(f"{outer}``{parameter}``") + self.data += f"{outer}``{parameter}``\n" if parameterdescs.get(parameter, '') != KernelDoc.undescribed: self.output_highlight(parameterdescs[parameter]) else: - print(f"{self.lineprefix}*undescribed*\n") - print() + self.data += f"{self.lineprefix}*undescribed*\n\n" + self.data += "\n" self.lineprefix = oldprefix self.out_section(args) @@ -458,14 +461,14 @@ class RestFormat(OutputFormat): name = args.get('typedef', '') ln = args.get('ln', 0) - print(f"\n\n.. c:type:: {name}\n") + self.data += f"\n\n.. c:type:: {name}\n\n" self.print_lineno(ln) self.lineprefix = " " self.output_highlight(args.get('purpose', '')) - print() + self.data += "\n" self.lineprefix = oldprefix self.out_section(args) @@ -482,7 +485,7 @@ class RestFormat(OutputFormat): parameterdescs = args.get('parameterdescs', {}) parameterdesc_start_lines = args.get('parameterdesc_start_lines', {}) - print(f"\n\n.. c:{dtype}:: {name}\n") + self.data += f"\n\n.. c:{dtype}:: {name}\n\n" self.print_lineno(ln) @@ -490,20 +493,20 @@ class RestFormat(OutputFormat): self.lineprefix += " " self.output_highlight(purpose) - print() + self.data += "\n" - print(".. container:: kernelindent\n") - print(f"{self.lineprefix}**Definition**::\n") + self.data += ".. container:: kernelindent\n\n" + self.data += f"{self.lineprefix}**Definition**::\n\n" self.lineprefix = self.lineprefix + " " declaration = declaration.replace("\t", self.lineprefix) - print(f"{self.lineprefix}{dtype} {name}" + ' {') - print(f"{declaration}{self.lineprefix}" + "};\n") + self.data += f"{self.lineprefix}{dtype} {name}" + ' {' + "\n" + self.data += f"{declaration}{self.lineprefix}" + "};\n\n" self.lineprefix = " " - print(f"{self.lineprefix}**Members**\n") + self.data += f"{self.lineprefix}**Members**\n\n" for parameter in parameterlist: if not parameter or parameter.startswith("#"): continue @@ -515,15 +518,15 @@ class RestFormat(OutputFormat): self.print_lineno(parameterdesc_start_lines.get(parameter_name, 0)) - print(f"{self.lineprefix}``{parameter}``") + self.data += f"{self.lineprefix}``{parameter}``\n" self.lineprefix = " " self.output_highlight(parameterdescs[parameter_name]) self.lineprefix = " " - print() + self.data += "\n" - print() + self.data += "\n" self.lineprefix = oldprefix self.out_section(args) @@ -576,19 +579,19 @@ class ManFormat(OutputFormat): line = Re(r"^\s*").sub("", line) if line and line[0] == ".": - print("\\&" + line) + self.data += "\\&" + line + "\n" else: - print(line) + self.data += line + "\n" def out_doc(self, fname, name, args): module = args.get('module') sectionlist = args.get('sectionlist', []) sections = args.get('sections', {}) - print(f'.TH "{module}" 9 "{module}" "{self.man_date}" "API Manual" LINUX') + self.data += f'.TH "{module}" 9 "{module}" "{self.man_date}" "API Manual" LINUX' + "\n" for section in sectionlist: - print(f'.SH "{section}"') + self.data += f'.SH "{section}"' + "\n" self.output_highlight(sections.get(section)) def out_function(self, fname, name, args): @@ -599,16 +602,16 @@ class ManFormat(OutputFormat): sectionlist = args.get('sectionlist', []) sections = args.get('sections', {}) - print(f'.TH "{args['function']}" 9 "{args['function']}" "{self.man_date}" "Kernel Hacker\'s Manual" LINUX') + self.data += f'.TH "{args['function']}" 9 "{args['function']}" "{self.man_date}" "Kernel Hacker\'s Manual" LINUX' + "\n" - print(".SH NAME") - print(f"{args['function']} \\- {args['purpose']}") + self.data += ".SH NAME\n" + self.data += f"{args['function']} \\- {args['purpose']}\n" - print(".SH SYNOPSIS") + self.data += ".SH SYNOPSIS\n" if args.get('functiontype', ''): - print(f'.B "{args['functiontype']}" {args['function']}') + self.data += f'.B "{args['functiontype']}" {args['function']}' + "\n" else: - print(f'.B "{args['function']}') + self.data += f'.B "{args['function']}' + "\n" count = 0 parenth = "(" @@ -621,25 +624,25 @@ class ManFormat(OutputFormat): dtype = args['parametertypes'].get(parameter, "") if function_pointer.match(dtype): # Pointer-to-function - print(f'".BI "{parenth}{function_pointer.group(1)}" " ") ({function_pointer.group(2)}){post}"') + self.data += f'".BI "{parenth}{function_pointer.group(1)}" " ") ({function_pointer.group(2)}){post}"' + "\n" else: dtype = Re(r'([^\*])$').sub(r'\1 ', dtype) - print(f'.BI "{parenth}{dtype}" "{post}"') + self.data += f'.BI "{parenth}{dtype}" "{post}"' + "\n" count += 1 parenth = "" if parameterlist: - print(".SH ARGUMENTS") + self.data += ".SH ARGUMENTS\n" for parameter in parameterlist: parameter_name = re.sub(r'\[.*', '', parameter) - print(f'.IP "{parameter}" 12') + self.data += f'.IP "{parameter}" 12' + "\n" self.output_highlight(parameterdescs.get(parameter_name, "")) for section in sectionlist: - print(f'.SH "{section.upper()}"') + self.data += f'.SH "{section.upper()}"' + "\n" self.output_highlight(sections[section]) def out_enum(self, fname, name, args): @@ -649,33 +652,33 @@ class ManFormat(OutputFormat): sectionlist = args.get('sectionlist', []) sections = args.get('sections', {}) - print(f'.TH "{args['module']}" 9 "enum {args['enum']}" "{self.man_date}" "API Manual" LINUX') + self.data += f'.TH "{args['module']}" 9 "enum {args['enum']}" "{self.man_date}" "API Manual" LINUX' + "\n" - print(".SH NAME") - print(f"enum {args['enum']} \\- {args['purpose']}") + self.data += ".SH NAME\n" + self.data += f"enum {args['enum']} \\- {args['purpose']}\n" - print(".SH SYNOPSIS") - print(f"enum {args['enum']}" + " {") + self.data += ".SH SYNOPSIS\n" + self.data += f"enum {args['enum']}" + " {\n" count = 0 for parameter in parameterlist: - print(f'.br\n.BI " {parameter}"') + self.data += f'.br\n.BI " {parameter}"' + "\n" if count == len(parameterlist) - 1: - print("\n};") + self.data += "\n};\n" else: - print(", \n.br") + self.data += ", \n.br\n" count += 1 - print(".SH Constants") + self.data += ".SH Constants\n" for parameter in parameterlist: parameter_name = Re(r'\[.*').sub('', parameter) - print(f'.IP "{parameter}" 12') + self.data += f'.IP "{parameter}" 12' + "\n" self.output_highlight(args['parameterdescs'].get(parameter_name, "")) for section in sectionlist: - print(f'.SH "{section}"') + self.data += f'.SH "{section}"' + "\n" self.output_highlight(sections[section]) def out_typedef(self, fname, name, args): @@ -685,13 +688,13 @@ class ManFormat(OutputFormat): sectionlist = args.get('sectionlist', []) sections = args.get('sections', {}) - print(f'.TH "{module}" 9 "{typedef}" "{self.man_date}" "API Manual" LINUX') + self.data += f'.TH "{module}" 9 "{typedef}" "{self.man_date}" "API Manual" LINUX' + "\n" - print(".SH NAME") - print(f"typedef {typedef} \\- {purpose}") + self.data += ".SH NAME\n" + self.data += f"typedef {typedef} \\- {purpose}\n" for section in sectionlist: - print(f'.SH "{section}"') + self.data += f'.SH "{section}"' + "\n" self.output_highlight(sections.get(section)) def out_struct(self, fname, name, args): @@ -705,20 +708,20 @@ class ManFormat(OutputFormat): sections = args.get('sections', {}) parameterdescs = args.get('parameterdescs', {}) - print(f'.TH "{module}" 9 "{struct_type} {struct_name}" "{self.man_date}" "API Manual" LINUX') + self.data += f'.TH "{module}" 9 "{struct_type} {struct_name}" "{self.man_date}" "API Manual" LINUX' + "\n" - print(".SH NAME") - print(f"{struct_type} {struct_name} \\- {purpose}") + self.data += ".SH NAME\n" + self.data += f"{struct_type} {struct_name} \\- {purpose}\n" # Replace tabs with two spaces and handle newlines declaration = definition.replace("\t", " ") declaration = Re(r"\n").sub('"\n.br\n.BI "', declaration) - print(".SH SYNOPSIS") - print(f"{struct_type} {struct_name} " + "{" + "\n.br") - print(f'.BI "{declaration}\n' + "};\n.br\n") + self.data += ".SH SYNOPSIS\n" + self.data += f"{struct_type} {struct_name} " + "{" + "\n.br\n" + self.data += f'.BI "{declaration}\n' + "};\n.br\n\n" - print(".SH Members") + self.data += ".SH Members\n" for parameter in parameterlist: if parameter.startswith("#"): continue @@ -728,9 +731,9 @@ class ManFormat(OutputFormat): if parameterdescs.get(parameter_name) == KernelDoc.undescribed: continue - print(f'.IP "{parameter}" 12') + self.data += f'.IP "{parameter}" 12' + "\n" self.output_highlight(parameterdescs.get(parameter_name)) for section in sectionlist: - print(f'.SH "{section}"') + self.data += f'.SH "{section}"' + "\n" self.output_highlight(sections.get(section)) -- 2.51.0 From 799b0d2a2a24f09f3800678f55ad48d8cc9c069d Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 8 Apr 2025 18:09:15 +0800 Subject: [PATCH 04/16] scripts/kernel-doc.py: move file lists to the parser function Instead of setting file lists at __init__ time, move it to the actual parsing function. This allows adding more files to be parsed in real time, by calling parse function multiple times. With the new way, the export_files logic was rewritten to avoid parsing twice EXPORT_SYMBOL for partial matches. Please notice that, with this logic, it can still read the same file twice when export_file is used. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/ab10bc94050406ce6536d4944b5d718ecd70812f.1744106242.git.mchehab+huawei@kernel.org --- scripts/kernel-doc.py | 7 +++---- scripts/lib/kdoc/kdoc_files.py | 37 ++++++++++++++++------------------ 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/scripts/kernel-doc.py b/scripts/kernel-doc.py index 63efec4b3f4b..e258a9df7f78 100755 --- a/scripts/kernel-doc.py +++ b/scripts/kernel-doc.py @@ -274,14 +274,13 @@ def main(): else: out_style = RestFormat() - kfiles = KernelFiles(files=args.files, verbose=args.verbose, + kfiles = KernelFiles(verbose=args.verbose, out_style=out_style, werror=args.werror, wreturn=args.wreturn, wshort_desc=args.wshort_desc, wcontents_before_sections=args.wcontents_before_sections, - modulename=args.modulename, - export_file=args.export_file) + modulename=args.modulename) - kfiles.parse() + kfiles.parse(args.files, export_file=args.export_file) for t in kfiles.msg(enable_lineno=args.enable_lineno, export=args.export, internal=args.internal, symbol=args.symbol, diff --git a/scripts/lib/kdoc/kdoc_files.py b/scripts/lib/kdoc/kdoc_files.py index 817ed98b2727..47dab46c89fe 100755 --- a/scripts/lib/kdoc/kdoc_files.py +++ b/scripts/lib/kdoc/kdoc_files.py @@ -124,7 +124,7 @@ class KernelFiles(): self.config.log.error("Cannot find file %s", fname) self.config.errors += 1 - def __init__(self, files=None, verbose=False, out_style=None, + def __init__(self, verbose=False, out_style=None, werror=False, wreturn=False, wshort_desc=False, wcontents_before_sections=False, logger=None, modulename=None, export_file=None): @@ -181,51 +181,48 @@ class KernelFiles(): self.config.src_tree = os.environ.get("SRCTREE", None) self.out_style = out_style - self.export_file = export_file # Initialize internal variables self.config.errors = 0 self.results = [] - self.file_list = files self.files = set() + self.export_files = set() - def parse(self): + def parse(self, file_list, export_file=None): """ Parse all files """ glob = GlobSourceFiles(srctree=self.config.src_tree) - # Let's use a set here to avoid duplicating files + # Prevent parsing the same file twice to speedup parsing and + # avoid reporting errors multiple times - for fname in glob.parse_files(self.file_list, self.file_not_found_cb): + for fname in glob.parse_files(file_list, self.file_not_found_cb): if fname in self.files: continue - self.files.add(fname) - res = self.parse_file(fname) - self.results.append((res.fname, res.entries)) - if not self.files: - sys.exit(1) + self.results.append((res.fname, res.entries)) + self.files.add(fname) # If a list of export files was provided, parse EXPORT_SYMBOL* - # from the ones not already parsed + # from files that weren't fully parsed - if self.export_file: - files = self.files + if not export_file: + return - glob = GlobSourceFiles(srctree=self.config.src_tree) + self.export_files |= self.files - for fname in glob.parse_files(self.export_file, - self.file_not_found_cb): - if fname not in files: - files.add(fname) + glob = GlobSourceFiles(srctree=self.config.src_tree) - self.process_export_file(fname) + for fname in glob.parse_files(export_file, self.file_not_found_cb): + if fname not in self.export_files: + self.process_export_file(fname) + self.export_files.add(fname) def out_msg(self, fname, name, arg): """ -- 2.51.0 From 0873e55433769210c0ba26227f0080dde408e15e Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 8 Apr 2025 18:09:16 +0800 Subject: [PATCH 05/16] scripts/kernel-doc.py: implement support for -no-doc-sections The venerable kernel-doc Perl script has a number of options that aren't properly documented. Among them, there is -no-doc-sections, which is used by the Sphinx extension. Implement support for it. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/06b18a32142b44d5ba8b41ac64a76c02b03b4969.1744106242.git.mchehab+huawei@kernel.org --- scripts/kernel-doc.py | 8 ++++++-- scripts/lib/kdoc/kdoc_files.py | 5 +++-- scripts/lib/kdoc/kdoc_output.py | 7 ++++++- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/scripts/kernel-doc.py b/scripts/kernel-doc.py index e258a9df7f78..90aacd17499a 100755 --- a/scripts/kernel-doc.py +++ b/scripts/kernel-doc.py @@ -239,10 +239,13 @@ def main(): sel_mut.add_argument("-s", "-function", "--symbol", action='append', help=FUNCTION_DESC) - # This one is valid for all 3 types of filter + # Those are valid for all 3 types of filter parser.add_argument("-n", "-nosymbol", "--nosymbol", action='append', help=NOSYMBOL_DESC) + parser.add_argument("-D", "-no-doc-sections", "--no-doc-sections", + action='store_true', help="Don't outputt DOC sections") + parser.add_argument("files", metavar="FILE", nargs="+", help=FILES_DESC) @@ -284,7 +287,8 @@ def main(): for t in kfiles.msg(enable_lineno=args.enable_lineno, export=args.export, internal=args.internal, symbol=args.symbol, - nosymbol=args.nosymbol): + nosymbol=args.nosymbol, + no_doc_sections=args.no_doc_sections): msg = t[1] if msg: print(msg) diff --git a/scripts/lib/kdoc/kdoc_files.py b/scripts/lib/kdoc/kdoc_files.py index 47dab46c89fe..4c04546a74fe 100755 --- a/scripts/lib/kdoc/kdoc_files.py +++ b/scripts/lib/kdoc/kdoc_files.py @@ -238,7 +238,7 @@ class KernelFiles(): return self.out_style.msg(fname, name, arg) def msg(self, enable_lineno=False, export=False, internal=False, - symbol=None, nosymbol=None): + symbol=None, nosymbol=None, no_doc_sections=False): """ Interacts over the kernel-doc results and output messages, returning kernel-doc markups on each interaction @@ -257,7 +257,8 @@ class KernelFiles(): self.out_style.set_config(self.config) self.out_style.set_filter(export, internal, symbol, nosymbol, - function_table, enable_lineno) + function_table, enable_lineno, + no_doc_sections) for fname, arg_tuple in self.results: msg = "" diff --git a/scripts/lib/kdoc/kdoc_output.py b/scripts/lib/kdoc/kdoc_output.py index fda07049ecf7..a246d213523c 100755 --- a/scripts/lib/kdoc/kdoc_output.py +++ b/scripts/lib/kdoc/kdoc_output.py @@ -70,6 +70,7 @@ class OutputFormat: self.symbol = None self.function_table = set() self.config = None + self.no_doc_sections = False self.data = "" @@ -77,7 +78,7 @@ class OutputFormat: self.config = config def set_filter(self, export, internal, symbol, nosymbol, function_table, - enable_lineno): + enable_lineno, no_doc_sections): """ Initialize filter variables according with the requested mode. @@ -87,6 +88,7 @@ class OutputFormat: """ self.enable_lineno = enable_lineno + self.no_doc_sections = no_doc_sections if symbol: self.out_mode = self.OUTPUT_INCLUDE @@ -117,6 +119,9 @@ class OutputFormat: def check_doc(self, name): """Check if DOC should be output""" + if self.no_doc_sections: + return False + if self.out_mode == self.OUTPUT_ALL: return True -- 2.51.0 From c3597ab27bc0e5eae23c74a76380000a0f8481e1 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 8 Apr 2025 18:09:17 +0800 Subject: [PATCH 06/16] scripts/kernel-doc.py: fix line number output With the Pyhton version, the actual output happens after parsing, from records stored at self.entries. Ensure that line numbers will be properly stored there and that they'll produce the desired results at the ReST output. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/5182a531d14b5fe9e1fc5da5f9dae05d66852a60.1744106242.git.mchehab+huawei@kernel.org --- scripts/lib/kdoc/kdoc_output.py | 13 +++++++------ scripts/lib/kdoc/kdoc_parser.py | 21 +++++++++++++++++---- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/scripts/lib/kdoc/kdoc_output.py b/scripts/lib/kdoc/kdoc_output.py index a246d213523c..6a7187980bec 100755 --- a/scripts/lib/kdoc/kdoc_output.py +++ b/scripts/lib/kdoc/kdoc_output.py @@ -255,7 +255,8 @@ class RestFormat(OutputFormat): def print_lineno(self, ln): """Outputs a line number""" - if self.enable_lineno and ln: + if self.enable_lineno and ln is not None: + ln += 1 self.data += f".. LINENO {ln}\n" def output_highlight(self, args): @@ -358,7 +359,7 @@ class RestFormat(OutputFormat): parameterdescs = args.get('parameterdescs', {}) parameterdesc_start_lines = args.get('parameterdesc_start_lines', {}) - ln = args.get('ln', 0) + ln = args.get('declaration_start_line', 0) count = 0 for parameter in parameterlist: @@ -375,11 +376,11 @@ class RestFormat(OutputFormat): if not func_macro: signature += ")" + self.print_lineno(ln) if args.get('typedef') or not args.get('functiontype'): self.data += f".. c:macro:: {args['function']}\n\n" if args.get('typedef'): - self.print_lineno(ln) self.data += " **Typedef**: " self.lineprefix = "" self.output_highlight(args.get('purpose', "")) @@ -434,7 +435,7 @@ class RestFormat(OutputFormat): name = args.get('enum', '') parameterlist = args.get('parameterlist', []) parameterdescs = args.get('parameterdescs', {}) - ln = args.get('ln', 0) + ln = args.get('declaration_start_line', 0) self.data += f"\n\n.. c:enum:: {name}\n\n" @@ -464,7 +465,7 @@ class RestFormat(OutputFormat): oldprefix = self.lineprefix name = args.get('typedef', '') - ln = args.get('ln', 0) + ln = args.get('declaration_start_line', 0) self.data += f"\n\n.. c:type:: {name}\n\n" @@ -484,7 +485,7 @@ class RestFormat(OutputFormat): purpose = args.get('purpose', "") declaration = args.get('definition', "") dtype = args.get('type', "struct") - ln = args.get('ln', 0) + ln = args.get('declaration_start_line', 0) parameterlist = args.get('parameterlist', []) parameterdescs = args.get('parameterdescs', {}) diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 3ce116595546..e8c86448d6b5 100755 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -276,7 +276,7 @@ class KernelDoc: self.entry.brcount = 0 self.entry.in_doc_sect = False - self.entry.declaration_start_line = ln + self.entry.declaration_start_line = ln + 1 def push_parameter(self, ln, decl_type, param, dtype, org_arg, declaration_name): @@ -806,8 +806,10 @@ class KernelDoc: parameterlist=self.entry.parameterlist, parameterdescs=self.entry.parameterdescs, parametertypes=self.entry.parametertypes, + parameterdesc_start_lines=self.entry.parameterdesc_start_lines, sectionlist=self.entry.sectionlist, sections=self.entry.sections, + section_start_lines=self.entry.section_start_lines, purpose=self.entry.declaration_purpose) def dump_enum(self, ln, proto): @@ -882,8 +884,10 @@ class KernelDoc: module=self.config.modulename, parameterlist=self.entry.parameterlist, parameterdescs=self.entry.parameterdescs, + parameterdesc_start_lines=self.entry.parameterdesc_start_lines, sectionlist=self.entry.sectionlist, sections=self.entry.sections, + section_start_lines=self.entry.section_start_lines, purpose=self.entry.declaration_purpose) def dump_declaration(self, ln, prototype): @@ -1054,8 +1058,10 @@ class KernelDoc: parameterlist=self.entry.parameterlist, parameterdescs=self.entry.parameterdescs, parametertypes=self.entry.parametertypes, + parameterdesc_start_lines=self.entry.parameterdesc_start_lines, sectionlist=self.entry.sectionlist, sections=self.entry.sections, + section_start_lines=self.entry.section_start_lines, purpose=self.entry.declaration_purpose, func_macro=func_macro) else: @@ -1067,8 +1073,10 @@ class KernelDoc: parameterlist=self.entry.parameterlist, parameterdescs=self.entry.parameterdescs, parametertypes=self.entry.parametertypes, + parameterdesc_start_lines=self.entry.parameterdesc_start_lines, sectionlist=self.entry.sectionlist, sections=self.entry.sections, + section_start_lines=self.entry.section_start_lines, purpose=self.entry.declaration_purpose, func_macro=func_macro) @@ -1112,8 +1120,10 @@ class KernelDoc: parameterlist=self.entry.parameterlist, parameterdescs=self.entry.parameterdescs, parametertypes=self.entry.parametertypes, + parameterdesc_start_lines=self.entry.parameterdesc_start_lines, sectionlist=self.entry.sectionlist, sections=self.entry.sections, + section_start_lines=self.entry.section_start_lines, purpose=self.entry.declaration_purpose) return @@ -1136,6 +1146,7 @@ class KernelDoc: module=self.entry.modulename, sectionlist=self.entry.sectionlist, sections=self.entry.sections, + section_start_lines=self.entry.section_start_lines, purpose=self.entry.declaration_purpose) return @@ -1168,7 +1179,7 @@ class KernelDoc: return # start a new entry - self.reset_state(ln + 1) + self.reset_state(ln) self.entry.in_doc_sect = False # next line is always the function name @@ -1281,7 +1292,7 @@ class KernelDoc: if r.match(line): self.dump_section() self.entry.section = self.section_default - self.entry.new_start_line = line + self.entry.new_start_line = ln self.entry.contents = "" if doc_sect.search(line): @@ -1619,7 +1630,9 @@ class KernelDoc: self.dump_section() self.output_declaration("doc", None, sectionlist=self.entry.sectionlist, - sections=self.entry.sections, module=self.config.modulename) + sections=self.entry.sections, + section_start_lines=self.entry.section_start_lines, + module=self.config.modulename) self.reset_state(ln) elif doc_content.search(line): -- 2.51.0 From 408269ae35d6b88d48477af56a2376ea05e619ca Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 8 Apr 2025 18:09:18 +0800 Subject: [PATCH 07/16] scripts/kernel-doc.py: fix handling of doc output check The filtering logic was seeking for the DOC name to check for symbols, but such data is stored only inside a section. Add it to the output_declaration, as it is quicker/easier to check the declaration name than to check inside each section. While here, make sure that the output for both ReST and man after filtering will be similar to what kernel-doc Perl version does. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/6d8b77af85295452c0191863ea1041f4195aeaaf.1744106242.git.mchehab+huawei@kernel.org --- scripts/lib/kdoc/kdoc_output.py | 29 ++++++++++++----------------- scripts/lib/kdoc/kdoc_parser.py | 3 ++- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/scripts/lib/kdoc/kdoc_output.py b/scripts/lib/kdoc/kdoc_output.py index 6a7187980bec..7a945dd80c9b 100755 --- a/scripts/lib/kdoc/kdoc_output.py +++ b/scripts/lib/kdoc/kdoc_output.py @@ -122,13 +122,13 @@ class OutputFormat: if self.no_doc_sections: return False + if name in self.nosymbol: + return False + if self.out_mode == self.OUTPUT_ALL: return True if self.out_mode == self.OUTPUT_INCLUDE: - if name in self.nosymbol: - return False - if name in self.function_table: return True @@ -154,15 +154,6 @@ class OutputFormat: return False - def check_function(self, fname, name, args): - return True - - def check_enum(self, fname, name, args): - return True - - def check_typedef(self, fname, name, args): - return True - def msg(self, fname, name, args): self.data = "" @@ -306,7 +297,7 @@ class RestFormat(OutputFormat): for line in output.strip("\n").split("\n"): self.data += self.lineprefix + line + "\n" - def out_section(self, args, out_reference=False): + def out_section(self, args, out_docblock=False): """ Outputs a block section. @@ -325,7 +316,7 @@ class RestFormat(OutputFormat): continue if not self.out_mode == self.OUTPUT_INCLUDE: - if out_reference: + if out_docblock: self.data += f".. _{section}:\n\n" if not self.symbol: @@ -339,8 +330,7 @@ class RestFormat(OutputFormat): def out_doc(self, fname, name, args): if not self.check_doc(name): return - - self.out_section(args, out_reference=True) + self.out_section(args, out_docblock=True) def out_function(self, fname, name, args): @@ -583,8 +573,10 @@ class ManFormat(OutputFormat): for line in contents.strip("\n").split("\n"): line = Re(r"^\s*").sub("", line) + if not line: + continue - if line and line[0] == ".": + if line[0] == ".": self.data += "\\&" + line + "\n" else: self.data += line + "\n" @@ -594,6 +586,9 @@ class ManFormat(OutputFormat): sectionlist = args.get('sectionlist', []) sections = args.get('sections', {}) + if not self.check_doc(name): + return + self.data += f'.TH "{module}" 9 "{module}" "{self.man_date}" "API Manual" LINUX' + "\n" for section in sectionlist: diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index e8c86448d6b5..74b311c8184c 100755 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -1198,6 +1198,7 @@ class KernelDoc: else: self.entry.section = doc_block.group(1) + self.entry.identifier = self.entry.section self.state = self.STATE_DOCBLOCK return @@ -1628,7 +1629,7 @@ class KernelDoc: if doc_end.search(line): self.dump_section() - self.output_declaration("doc", None, + self.output_declaration("doc", self.entry.identifier, sectionlist=self.entry.sectionlist, sections=self.entry.sections, section_start_lines=self.entry.section_start_lines, -- 2.51.0 From 9235ec5e2bf8ccd3f3ef30bbb1811bf9ddb3ef08 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 8 Apr 2025 18:09:19 +0800 Subject: [PATCH 08/16] scripts/kernel-doc.py: properly handle out_section for ReST There is a difference at the way DOC sections are output with the include mode. Handle such difference properly. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/935d00c6a7c45b25a8be72fad6183fe5a8476cd2.1744106242.git.mchehab+huawei@kernel.org --- scripts/lib/kdoc/kdoc_output.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/lib/kdoc/kdoc_output.py b/scripts/lib/kdoc/kdoc_output.py index 7a945dd80c9b..d0c8cedb0ea5 100755 --- a/scripts/lib/kdoc/kdoc_output.py +++ b/scripts/lib/kdoc/kdoc_output.py @@ -315,12 +315,12 @@ class RestFormat(OutputFormat): if section in self.nosymbol: continue - if not self.out_mode == self.OUTPUT_INCLUDE: - if out_docblock: + if out_docblock: + if not self.out_mode == self.OUTPUT_INCLUDE: self.data += f".. _{section}:\n\n" - - if not self.symbol: self.data += f'{self.lineprefix}**{section}**\n\n' + else: + self.data += f'{self.lineprefix}**{section}**\n\n' self.print_lineno(section_start_lines.get(section, 0)) self.output_highlight(sections[section]) -- 2.51.0 From 9cbc2d3b137bfdb7937265c46e9e5d7e72952841 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 8 Apr 2025 18:09:20 +0800 Subject: [PATCH 09/16] scripts/kernel-doc.py: postpone warnings to the output plugin We don't want to have warnings displayed for symbols that weren't output. So, postpone warnings print to the output plugin, where symbol output is validated. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/e6344711e390cf22af02a56bb5dd51ca67c0afb6.1744106242.git.mchehab+huawei@kernel.org --- scripts/lib/kdoc/kdoc_output.py | 24 +++++++++++++++---- scripts/lib/kdoc/kdoc_parser.py | 41 ++++++++++++++++----------------- 2 files changed, 39 insertions(+), 26 deletions(-) diff --git a/scripts/lib/kdoc/kdoc_output.py b/scripts/lib/kdoc/kdoc_output.py index d0c8cedb0ea5..6582d1f64d1e 100755 --- a/scripts/lib/kdoc/kdoc_output.py +++ b/scripts/lib/kdoc/kdoc_output.py @@ -116,7 +116,16 @@ class OutputFormat: return block - def check_doc(self, name): + def out_warnings(self, args): + warnings = args.get('warnings', []) + + for warning, log_msg in warnings: + if warning: + self.config.log.warning(log_msg) + else: + self.config.log.info(log_msg) + + def check_doc(self, name, args): """Check if DOC should be output""" if self.no_doc_sections: @@ -126,19 +135,22 @@ class OutputFormat: return False if self.out_mode == self.OUTPUT_ALL: + self.out_warnings(args) return True if self.out_mode == self.OUTPUT_INCLUDE: if name in self.function_table: + self.out_warnings(args) return True return False - def check_declaration(self, dtype, name): + def check_declaration(self, dtype, name, args): if name in self.nosymbol: return False if self.out_mode == self.OUTPUT_ALL: + self.out_warnings(args) return True if self.out_mode in [self.OUTPUT_INCLUDE, self.OUTPUT_EXPORTED]: @@ -147,9 +159,11 @@ class OutputFormat: if self.out_mode == self.OUTPUT_INTERNAL: if dtype != "function": + self.out_warnings(args) return True if name not in self.function_table: + self.out_warnings(args) return True return False @@ -163,7 +177,7 @@ class OutputFormat: self.out_doc(fname, name, args) return self.data - if not self.check_declaration(dtype, name): + if not self.check_declaration(dtype, name, args): return self.data if dtype == "function": @@ -328,7 +342,7 @@ class RestFormat(OutputFormat): self.data += "\n" def out_doc(self, fname, name, args): - if not self.check_doc(name): + if not self.check_doc(name, args): return self.out_section(args, out_docblock=True) @@ -586,7 +600,7 @@ class ManFormat(OutputFormat): sectionlist = args.get('sectionlist', []) sections = args.get('sections', {}) - if not self.check_doc(name): + if not self.check_doc(name, args): return self.data += f'.TH "{module}" 9 "{module}" "{self.man_date}" "API Manual" LINUX' + "\n" diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 74b311c8184c..3698ef625367 100755 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -131,23 +131,23 @@ class KernelDoc: # Place all potential outputs into an array self.entries = [] - def show_warnings(self, dtype, declaration_name): # pylint: disable=W0613 - """ - Allow filtering out warnings - """ - - # TODO: implement it - - return True - # TODO: rename to emit_message def emit_warning(self, ln, msg, warning=True): """Emit a message""" + log_msg = f"{self.fname}:{ln} {msg}" + + if self.entry: + # Delegate warning output to output logic, as this way it + # will report warnings/info only for symbols that are output + + self.entry.warnings.append((warning, log_msg)) + return + if warning: - self.config.log.warning("%s:%d %s", self.fname, ln, msg) + self.config.log.warning(log_msg) else: - self.config.log.info("%s:%d %s", self.fname, ln, msg) + self.config.log.info(log_msg) def dump_section(self, start_new=True): """ @@ -221,10 +221,9 @@ class KernelDoc: # For now, we're keeping the same name of the function just to make # easier to compare the source code of both scripts - if "declaration_start_line" not in args: - args["declaration_start_line"] = self.entry.declaration_start_line - + args["declaration_start_line"] = self.entry.declaration_start_line args["type"] = dtype + args["warnings"] = self.entry.warnings # TODO: use colletions.OrderedDict @@ -257,6 +256,8 @@ class KernelDoc: self.entry.struct_actual = "" self.entry.prototype = "" + self.entry.warnings = [] + self.entry.parameterlist = [] self.entry.parameterdescs = {} self.entry.parametertypes = {} @@ -328,7 +329,7 @@ class KernelDoc: if param not in self.entry.parameterdescs and not param.startswith("#"): self.entry.parameterdescs[param] = self.undescribed - if self.show_warnings(dtype, declaration_name) and "." not in param: + if "." not in param: if decl_type == 'function': dname = f"{decl_type} parameter" else: @@ -868,16 +869,14 @@ class KernelDoc: self.entry.parameterlist.append(arg) if arg not in self.entry.parameterdescs: self.entry.parameterdescs[arg] = self.undescribed - if self.show_warnings("enum", declaration_name): - self.emit_warning(ln, - f"Enum value '{arg}' not described in enum '{declaration_name}'") + self.emit_warning(ln, + f"Enum value '{arg}' not described in enum '{declaration_name}'") member_set.add(arg) for k in self.entry.parameterdescs: if k not in member_set: - if self.show_warnings("enum", declaration_name): - self.emit_warning(ln, - f"Excess enum value '%{k}' description in '{declaration_name}'") + self.emit_warning(ln, + f"Excess enum value '%{k}' description in '{declaration_name}'") self.output_declaration('enum', declaration_name, enum=declaration_name, -- 2.51.0 From 02df8e3b333c3d8550a22ffdfc969caec8462df9 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 8 Apr 2025 18:09:21 +0800 Subject: [PATCH 10/16] docs: add a .pylintrc file with sys path for docs scripts The docs scripts that are used by Documentation/sphinx are using scripts/lib/* directories to place classes that will be used by both extensions and scripts. When pylint is used, it needs to identify the path where such scripts are, otherwise it will bail out. Add a simple RC file placing the location of such files. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/7b3c8a932c50ae52ce4c848676602b46d1d4a8f9.1744106242.git.mchehab+huawei@kernel.org --- .pylintrc | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .pylintrc diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 000000000000..30b8ae1659f8 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,2 @@ +[MASTER] +init-hook='import sys; sys.path += ["scripts/lib/kdoc", "scripts/lib/abi"]' -- 2.51.0 From 668b9d1dceb86b570ff28d913e8464ba62f57e91 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 8 Apr 2025 18:09:22 +0800 Subject: [PATCH 11/16] docs: sphinx: kerneldoc: verbose kernel-doc command if V=1 It is useful to know what kernel-doc command was used during document build time, as it allows one to check the output the same way as Sphinx extension does. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/a2f01590814b111e138f278e8a721024fdf2d445.1744106242.git.mchehab+huawei@kernel.org --- Documentation/sphinx/kerneldoc.py | 34 +++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/Documentation/sphinx/kerneldoc.py b/Documentation/sphinx/kerneldoc.py index 39ddae6ae7dd..d206eb2be10a 100644 --- a/Documentation/sphinx/kerneldoc.py +++ b/Documentation/sphinx/kerneldoc.py @@ -43,6 +43,29 @@ from sphinx.util import logging __version__ = '1.0' +def cmd_str(cmd): + """ + Helper function to output a command line that can be used to produce + the same records via command line. Helpful to debug troubles at the + script. + """ + + cmd_line = "" + + for w in cmd: + if w == "" or " " in w: + esc_cmd = "'" + w + "'" + else: + esc_cmd = w + + if cmd_line: + cmd_line += " " + esc_cmd + continue + else: + cmd_line = esc_cmd + + return cmd_line + class KernelDocDirective(Directive): """Extract kernel-doc comments from the specified file""" required_argument = 1 @@ -57,6 +80,7 @@ class KernelDocDirective(Directive): } has_content = False logger = logging.getLogger('kerneldoc') + verbose = 0 def run(self): env = self.state.document.settings.env @@ -65,6 +89,13 @@ class KernelDocDirective(Directive): filename = env.config.kerneldoc_srctree + '/' + self.arguments[0] export_file_patterns = [] + verbose = os.environ.get("V") + if verbose: + try: + self.verbose = int(verbose) + except ValueError: + pass + # Tell sphinx of the dependency env.note_dependency(os.path.abspath(filename)) @@ -104,6 +135,9 @@ class KernelDocDirective(Directive): cmd += [filename] + if self.verbose >= 1: + print(cmd_str(cmd)) + try: self.logger.verbose("calling kernel-doc '%s'" % (" ".join(cmd))) -- 2.51.0 From 01c43355255e1f0bba8677fc66facc0047a23242 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 8 Apr 2025 18:09:23 +0800 Subject: [PATCH 12/16] docs: sphinx: kerneldoc: ignore "\" characters from options Documentation/driver-api/infiniband.rst has a kernel-doc tag with "\" characters at the end: .. kernel-doc:: drivers/infiniband/ulp/iser/iscsi_iser.c :functions: iscsi_iser_pdu_alloc iser_initialize_task_headers \ iscsi_iser_task_init iscsi_iser_mtask_xmit iscsi_iser_task_xmit \ iscsi_iser_cleanup_task iscsi_iser_check_protection \ iscsi_iser_conn_create iscsi_iser_conn_bind \ iscsi_iser_conn_start iscsi_iser_conn_stop \ iscsi_iser_session_destroy iscsi_iser_session_create \ iscsi_iser_set_param iscsi_iser_ep_connect iscsi_iser_ep_poll \ iscsi_iser_ep_disconnect This is not handled well, as the "\" strings will be just stored inside Sphinx options. While the actual problem deserves being fixed, better to relax the keneldoc.py extension to silently strip "\" from the end of strings, as otherwise this may cause troubles when preparing arguments to be executed by kernel-doc. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/4c652d6c57b20500c135b95294e554d9e9a97f42.1744106242.git.mchehab+huawei@kernel.org --- Documentation/sphinx/kerneldoc.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Documentation/sphinx/kerneldoc.py b/Documentation/sphinx/kerneldoc.py index d206eb2be10a..344789ed9ea2 100644 --- a/Documentation/sphinx/kerneldoc.py +++ b/Documentation/sphinx/kerneldoc.py @@ -118,6 +118,10 @@ class KernelDocDirective(Directive): identifiers = self.options.get('identifiers').split() if identifiers: for i in identifiers: + i = i.rstrip("\\").strip() + if not i: + continue + cmd += ['-function', i] else: cmd += ['-no-doc-sections'] @@ -126,9 +130,17 @@ class KernelDocDirective(Directive): no_identifiers = self.options.get('no-identifiers').split() if no_identifiers: for i in no_identifiers: + i = i.rstrip("\\").strip() + if not i: + continue + cmd += ['-nosymbol', i] for pattern in export_file_patterns: + pattern = pattern.rstrip("\\").strip() + if not pattern: + continue + for f in glob.glob(env.config.kerneldoc_srctree + '/' + pattern): env.note_dependency(os.path.abspath(f)) cmd += ['-export-file', f] -- 2.51.0 From feec610725e38e96dca0f77f1fc388f59ffa616b Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 8 Apr 2025 18:09:24 +0800 Subject: [PATCH 13/16] docs: sphinx: kerneldoc: use kernel-doc.py script Switch to the new version when producing documentation. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/a81d8db099d9cef5161deaef40ac9056bf9802a3.1744106242.git.mchehab+huawei@kernel.org --- Documentation/Makefile | 2 +- Documentation/conf.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/Makefile b/Documentation/Makefile index 63094646df28..c022b97c487e 100644 --- a/Documentation/Makefile +++ b/Documentation/Makefile @@ -60,7 +60,7 @@ endif #HAVE_LATEXMK # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter -KERNELDOC = $(srctree)/scripts/kernel-doc +KERNELDOC = $(srctree)/scripts/kernel-doc.py KERNELDOC_CONF = -D kerneldoc_srctree=$(srctree) -D kerneldoc_bin=$(KERNELDOC) ALLSPHINXOPTS = $(KERNELDOC_CONF) $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) ifneq ($(wildcard $(srctree)/.config),) diff --git a/Documentation/conf.py b/Documentation/conf.py index 3dad1f90b098..b126f6760b5f 100644 --- a/Documentation/conf.py +++ b/Documentation/conf.py @@ -540,7 +540,7 @@ pdf_documents = [ # kernel-doc extension configuration for running Sphinx directly (e.g. by Read # the Docs). In a normal build, these are supplied from the Makefile via command # line arguments. -kerneldoc_bin = '../scripts/kernel-doc' +kerneldoc_bin = '../scripts/kernel-doc.py' kerneldoc_srctree = '..' # ------------------------------------------------------------------------------ -- 2.51.0 From 43ecfe6bc2ae11f99fb4b812c014339c2d6a221a Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 8 Apr 2025 18:09:25 +0800 Subject: [PATCH 14/16] scripts/kernel-doc.py: Set an output format for --none Now that warnings output is deferred to the output plugin, we need to have an output style for none as well. So, use the OutputFormat base class on such cases. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/caa1089e16f4609f792ff26731ad9e9c3a6f6b1d.1744106242.git.mchehab+huawei@kernel.org --- scripts/lib/kdoc/kdoc_files.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/lib/kdoc/kdoc_files.py b/scripts/lib/kdoc/kdoc_files.py index 4c04546a74fe..dd3dbe87520b 100755 --- a/scripts/lib/kdoc/kdoc_files.py +++ b/scripts/lib/kdoc/kdoc_files.py @@ -20,6 +20,7 @@ from datetime import datetime from dateutil import tz from kdoc_parser import KernelDoc +from kdoc_output import OutputFormat class GlobSourceFiles: @@ -138,6 +139,9 @@ class KernelFiles(): if not modulename: modulename = "Kernel API" + if out_style is None: + out_style = OutputFormat() + dt = datetime.now() if os.environ.get("KBUILD_BUILD_TIMESTAMP", None): # use UTC TZ -- 2.51.0 From 485f6f7960c468d9e27665f61517dc5fc097ea98 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 8 Apr 2025 18:09:26 +0800 Subject: [PATCH 15/16] scripts/kernel-doc.py: adjust some coding style issues Make pylint happier by adding some missing documentation and addressing a couple of pylint warnings. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/0f9d5473105e4c09c6c41e3db72cc63f1d4d55f9.1744106242.git.mchehab+huawei@kernel.org --- scripts/kernel-doc.py | 12 ++++---- scripts/lib/kdoc/kdoc_files.py | 4 +-- scripts/lib/kdoc/kdoc_output.py | 50 ++++++++++++++++++++++++++------- scripts/lib/kdoc/kdoc_parser.py | 30 +++++--------------- scripts/lib/kdoc/kdoc_re.py | 3 +- 5 files changed, 57 insertions(+), 42 deletions(-) mode change 100755 => 100644 scripts/lib/kdoc/kdoc_files.py diff --git a/scripts/kernel-doc.py b/scripts/kernel-doc.py index 90aacd17499a..eca7e34f9d03 100755 --- a/scripts/kernel-doc.py +++ b/scripts/kernel-doc.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: GPL-2.0 # Copyright(c) 2025: Mauro Carvalho Chehab . # -# pylint: disable=C0103 +# pylint: disable=C0103,R0915 # # Converted from the kernel-doc script originally written in Perl # under GPLv2, copyrighted since 1998 by the following authors: @@ -165,6 +165,8 @@ neither here nor at the original Perl script. class MsgFormatter(logging.Formatter): + """Helper class to format warnings on a similar way to kernel-doc.pl""" + def format(self, record): record.levelname = record.levelname.capitalize() return logging.Formatter.format(self, record) @@ -241,7 +243,7 @@ def main(): # Those are valid for all 3 types of filter parser.add_argument("-n", "-nosymbol", "--nosymbol", action='append', - help=NOSYMBOL_DESC) + help=NOSYMBOL_DESC) parser.add_argument("-D", "-no-doc-sections", "--no-doc-sections", action='store_true', help="Don't outputt DOC sections") @@ -286,9 +288,9 @@ def main(): kfiles.parse(args.files, export_file=args.export_file) for t in kfiles.msg(enable_lineno=args.enable_lineno, export=args.export, - internal=args.internal, symbol=args.symbol, - nosymbol=args.nosymbol, - no_doc_sections=args.no_doc_sections): + internal=args.internal, symbol=args.symbol, + nosymbol=args.nosymbol, + no_doc_sections=args.no_doc_sections): msg = t[1] if msg: print(msg) diff --git a/scripts/lib/kdoc/kdoc_files.py b/scripts/lib/kdoc/kdoc_files.py old mode 100755 new mode 100644 index dd3dbe87520b..e2221db7022a --- a/scripts/lib/kdoc/kdoc_files.py +++ b/scripts/lib/kdoc/kdoc_files.py @@ -4,8 +4,6 @@ # # pylint: disable=R0903,R0913,R0914,R0917 -# TODO: implement warning filtering - """ Parse lernel-doc tags on multiple kernel source files. """ @@ -128,7 +126,7 @@ class KernelFiles(): def __init__(self, verbose=False, out_style=None, werror=False, wreturn=False, wshort_desc=False, wcontents_before_sections=False, - logger=None, modulename=None, export_file=None): + logger=None, modulename=None): """ Initialize startup variables and parse all files """ diff --git a/scripts/lib/kdoc/kdoc_output.py b/scripts/lib/kdoc/kdoc_output.py index 6582d1f64d1e..7f84bf12f1e1 100755 --- a/scripts/lib/kdoc/kdoc_output.py +++ b/scripts/lib/kdoc/kdoc_output.py @@ -2,9 +2,7 @@ # SPDX-License-Identifier: GPL-2.0 # Copyright(c) 2025: Mauro Carvalho Chehab . # -# pylint: disable=C0301,R0911,R0912,R0913,R0914,R0915,R0917 - -# TODO: implement warning filtering +# pylint: disable=C0301,R0902,R0911,R0912,R0913,R0914,R0915,R0917 """ Implement output filters to print kernel-doc documentation. @@ -52,6 +50,11 @@ type_member_func = type_member + Re(r"\(\)", cache=False) class OutputFormat: + """ + Base class for OutputFormat. If used as-is, it means that only + warnings will be displayed. + """ + # output mode. OUTPUT_ALL = 0 # output all symbols and doc sections OUTPUT_INCLUDE = 1 # output only specified symbols @@ -75,6 +78,10 @@ class OutputFormat: self.data = "" def set_config(self, config): + """ + Setup global config variables used by both parser and output. + """ + self.config = config def set_filter(self, export, internal, symbol, nosymbol, function_table, @@ -117,6 +124,10 @@ class OutputFormat: return block def out_warnings(self, args): + """ + Output warnings for identifiers that will be displayed. + """ + warnings = args.get('warnings', []) for warning, log_msg in warnings: @@ -146,6 +157,11 @@ class OutputFormat: return False def check_declaration(self, dtype, name, args): + """ + Checks if a declaration should be output or not based on the + filtering criteria. + """ + if name in self.nosymbol: return False @@ -169,6 +185,10 @@ class OutputFormat: return False def msg(self, fname, name, args): + """ + Handles a single entry from kernel-doc parser + """ + self.data = "" dtype = args.get('type', "") @@ -203,24 +223,25 @@ class OutputFormat: return None # Virtual methods to be overridden by inherited classes + # At the base class, those do nothing. def out_doc(self, fname, name, args): - pass + """Outputs a DOC block""" def out_function(self, fname, name, args): - pass + """Outputs a function""" def out_enum(self, fname, name, args): - pass + """Outputs an enum""" def out_typedef(self, fname, name, args): - pass + """Outputs a typedef""" def out_struct(self, fname, name, args): - pass + """Outputs a struct""" class RestFormat(OutputFormat): - # """Consts and functions used by ReST output""" + """Consts and functions used by ReST output""" highlights = [ (type_constant, r"``\1``"), @@ -265,6 +286,11 @@ class RestFormat(OutputFormat): self.data += f".. LINENO {ln}\n" def output_highlight(self, args): + """ + Outputs a C symbol that may require being converted to ReST using + the self.highlights variable + """ + input_text = args output = "" in_literal = False @@ -579,6 +605,10 @@ class ManFormat(OutputFormat): self.man_date = dt.strftime("%B %Y") def output_highlight(self, block): + """ + Outputs a C symbol that may require being highlighted with + self.highlights variable using troff syntax + """ contents = self.highlight_block(block) @@ -601,7 +631,7 @@ class ManFormat(OutputFormat): sections = args.get('sections', {}) if not self.check_doc(name, args): - return + return self.data += f'.TH "{module}" 9 "{module}" "{self.man_date}" "API Manual" LINUX' + "\n" diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index 3698ef625367..dcb9515fc40b 100755 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -131,7 +131,7 @@ class KernelDoc: # Place all potential outputs into an array self.entries = [] - # TODO: rename to emit_message + # TODO: rename to emit_message after removal of kernel-doc.pl def emit_warning(self, ln, msg, warning=True): """Emit a message""" @@ -157,19 +157,6 @@ class KernelDoc: name = self.entry.section contents = self.entry.contents - # TODO: we can prevent dumping empty sections here with: - # - # if self.entry.contents.strip("\n"): - # if start_new: - # self.entry.section = self.section_default - # self.entry.contents = "" - # - # return - # - # But, as we want to be producing the same output of the - # venerable kernel-doc Perl tool, let's just output everything, - # at least for now - if type_param.match(name): name = type_param.group(1) @@ -205,7 +192,7 @@ class KernelDoc: self.entry.section = self.section_default self.entry.contents = "" - # TODO: rename it to store_declaration + # TODO: rename it to store_declaration after removal of kernel-doc.pl def output_declaration(self, dtype, name, **args): """ Stores the entry into an entry array. @@ -225,13 +212,13 @@ class KernelDoc: args["type"] = dtype args["warnings"] = self.entry.warnings - # TODO: use colletions.OrderedDict + # TODO: use colletions.OrderedDict to remove sectionlist sections = args.get('sections', {}) sectionlist = args.get('sectionlist', []) # Drop empty sections - # TODO: improve it to emit warnings + # TODO: improve empty sections logic to emit warnings for section in ["Description", "Return"]: if section in sectionlist: if not sections[section].rstrip(): @@ -636,7 +623,9 @@ class KernelDoc: # Replace macros # - # TODO: it is better to also move those to the NestedMatch logic, + # TODO: use NestedMatch for FOO($1, $2, ...) matches + # + # it is better to also move those to the NestedMatch logic, # to ensure that parenthesis will be properly matched. (Re(r'__ETHTOOL_DECLARE_LINK_MODE_MASK\s*\(([^\)]+)\)', re.S), r'DECLARE_BITMAP(\1, __ETHTOOL_LINK_MODE_MASK_NBITS)'), @@ -906,7 +895,6 @@ class KernelDoc: self.dump_struct(ln, prototype) return - # TODO: handle other types self.output_declaration(self.entry.decl_type, prototype, entry=self.entry) @@ -1680,10 +1668,6 @@ class KernelDoc: self.st_inline_name[self.inline_doc_state], line) - # TODO: not all states allow EXPORT_SYMBOL*, so this - # can be optimized later on to speedup parsing - self.process_export(self.config.function_table, line) - # Hand this line to the appropriate state handler if self.state == self.STATE_NORMAL: self.process_normal(ln, line) diff --git a/scripts/lib/kdoc/kdoc_re.py b/scripts/lib/kdoc/kdoc_re.py index 512b6521e79d..d28485ff94d6 100755 --- a/scripts/lib/kdoc/kdoc_re.py +++ b/scripts/lib/kdoc/kdoc_re.py @@ -131,7 +131,8 @@ class NestedMatch: will ignore the search string. """ - # TODO: + # TODO: make NestedMatch handle multiple match groups + # # Right now, regular expressions to match it are defined only up to # the start delimiter, e.g.: # -- 2.51.0 From 78ea748f7978d39a6ee29897d3bd32e6208f74ac Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 8 Apr 2025 18:09:27 +0800 Subject: [PATCH 16/16] scripts/lib/kdoc/kdoc_parser.py: fix Python compat with < v3.13 - str.replace count was introduced only in Python 3.13; - before Python 3.13, f-string dict arguments can't use the same delimiter of the main string. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/e2b8e8361294558dae09236e4b8fbea5d86be5a3.1744106242.git.mchehab+huawei@kernel.org --- scripts/lib/kdoc/kdoc_output.py | 8 ++++---- scripts/lib/kdoc/kdoc_parser.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/lib/kdoc/kdoc_output.py b/scripts/lib/kdoc/kdoc_output.py index 7f84bf12f1e1..e0ed79e4d985 100755 --- a/scripts/lib/kdoc/kdoc_output.py +++ b/scripts/lib/kdoc/kdoc_output.py @@ -647,16 +647,16 @@ class ManFormat(OutputFormat): sectionlist = args.get('sectionlist', []) sections = args.get('sections', {}) - self.data += f'.TH "{args['function']}" 9 "{args['function']}" "{self.man_date}" "Kernel Hacker\'s Manual" LINUX' + "\n" + self.data += f'.TH "{args["function"]}" 9 "{args["function"]}" "{self.man_date}" "Kernel Hacker\'s Manual" LINUX' + "\n" self.data += ".SH NAME\n" self.data += f"{args['function']} \\- {args['purpose']}\n" self.data += ".SH SYNOPSIS\n" if args.get('functiontype', ''): - self.data += f'.B "{args['functiontype']}" {args['function']}' + "\n" + self.data += f'.B "{args["functiontype"]}" {args["function"]}' + "\n" else: - self.data += f'.B "{args['function']}' + "\n" + self.data += f'.B "{args["function"]}' + "\n" count = 0 parenth = "(" @@ -697,7 +697,7 @@ class ManFormat(OutputFormat): sectionlist = args.get('sectionlist', []) sections = args.get('sections', {}) - self.data += f'.TH "{args['module']}" 9 "enum {args['enum']}" "{self.man_date}" "API Manual" LINUX' + "\n" + self.data += f'.TH "{args["module"]}" 9 "enum {args["enum"]}" "{self.man_date}" "API Manual" LINUX' + "\n" self.data += ".SH NAME\n" self.data += f"enum {args['enum']} \\- {args['purpose']}\n" diff --git a/scripts/lib/kdoc/kdoc_parser.py b/scripts/lib/kdoc/kdoc_parser.py index dcb9515fc40b..e48ed128ca04 100755 --- a/scripts/lib/kdoc/kdoc_parser.py +++ b/scripts/lib/kdoc/kdoc_parser.py @@ -1444,9 +1444,9 @@ class KernelDoc: r = Re(r'long\s+(sys_.*?),') if r.search(proto): - proto = proto.replace(',', '(', count=1) + proto = Re(',').sub('(', proto, count=1) elif is_void: - proto = proto.replace(')', '(void)', count=1) + proto = Re(r'\)').sub('(void)', proto, count=1) # Now delete all of the odd-numbered commas in the proto # so that argument types & names don't have a comma between them -- 2.51.0