Messages in this thread Patch in this message | | | Date | Fri, 31 Jul 2020 14:21:41 -0700 | Subject | [PATCH] gen_compile_commands: Add support for separate kbuild output directory | From | Peter Kalauskas <> |
| |
Add support for builds that use an output directory different than the kernel source tree (e.g. make O=/tmp/kernel-obj). This also introduces support for .cmd files that use absolute paths.
Previously, gen_compile_commands.py only supported builds where the kernel source tree and the output directory were the same: $ make defconfig $ make -j32 $ ./scripts/gen_compile_commands.py
gen_compile_commands.py had flags to support out of tree use, but the generated compile_commands.json file still assumed that the source tree and kbuild output directory were the same.
Now, the following cases are supported as well:
- Absolute output path: $ mkdir /tmp/kernel-obj $ make O=/tmp/kernel-obj defconfig $ make O=/tmp/kernel-obj -j32 $ ./scripts/gen_compile_commands.py -k /tmp/kernel-obj
- Relative output path: $ mkdir kernel-obj $ make O=kernel-obj/ defconfig $ make O=kernel-obj/ -j32 $ ./scripts/gen_compile_commands.py -k kernel-obj
The new argument, -k, is introduced in a way that makes the script backward compatible with how -d was previously used.
Signed-off-by: Peter Kalauskas <peskal@google.com> --- scripts/gen_compile_commands.py | 112 ++++++++++++++++++++++---------- 1 file changed, 77 insertions(+), 35 deletions(-)
diff --git scripts/gen_compile_commands.py scripts/gen_compile_commands.py index c458696ef3a7..cd3b80bd1942 100755 --- scripts/gen_compile_commands.py +++ scripts/gen_compile_commands.py @@ -31,16 +31,24 @@ def parse_arguments(): Returns: log_level: A logging level to filter log output. - directory: The directory to search for .cmd files. + source_directory: The directory of the kernel source tree. + kbuild_output_directory: The directory to search for .cmd files. output: Where to write the compile-commands JSON file. """ usage = 'Creates a compile_commands.json database from kernel .cmd files' parser = argparse.ArgumentParser(description=usage) - directory_help = ('Path to the kernel source directory to search ' + directory_help = ('Path to the kernel source directory ' '(defaults to the working directory)') parser.add_argument('-d', '--directory', type=str, help=directory_help) + kbuild_output_directory_help = ( + 'Path to the kernel output directory to search for .cmd files ' + '(defaults to match --directory)') + parser.add_argument( + '-k', '--kbuild-output-directory', type=str, + help=kbuild_output_directory_help) + output_help = ('The location to write compile_commands.json (defaults to ' 'compile_commands.json in the search directory)') parser.add_argument('-o', '--output', type=str, help=output_help) @@ -58,58 +66,91 @@ def parse_arguments(): if log_level not in _VALID_LOG_LEVELS: raise ValueError('%s is not a valid log level' % log_level) - directory = args.directory or os.getcwd() - output = args.output or os.path.join(directory, _DEFAULT_OUTPUT) - directory = os.path.abspath(directory) + source_directory = args.directory or os.getcwd() + kbuild_output_directory = args.kbuild_output_directory or source_directory + output = args.output or os.path.join(source_directory, _DEFAULT_OUTPUT) + source_directory = os.path.abspath(source_directory) + kbuild_output_directory = os.path.abspath(kbuild_output_directory) - return log_level, directory, output + return log_level, source_directory, kbuild_output_directory, output -def process_line(root_directory, file_directory, command_prefix, relative_path): +def process_line(src_dir, kbuild_out_dir, file_dir, cmd_prefix, file_path): """Extracts information from a .cmd line and creates an entry from it. Args: - root_directory: The directory that was searched for .cmd files. Usually + src_dir: The directory of the kernel source tree. + kbuild_out_dir: The directory that was searched for .cmd files. Usually used directly in the "directory" entry in compile_commands.json. - file_directory: The path to the directory the .cmd file was found in. - command_prefix: The extracted command line, up to the last element. - relative_path: The .c file from the end of the extracted command. - Usually relative to root_directory, but sometimes relative to - file_directory and sometimes neither. + file_dir: The path to the directory the .cmd file was found in. + cmd_prefix: The extracted command line, up to the last element. + file_path: The .c file from the end of the extracted command. + Usually relative to kbuild_out_dir, but sometimes relative to + src_dir and sometimes neither. Returns: An entry to append to compile_commands. Raises: - ValueError: Could not find the extracted file based on relative_path and - root_directory or file_directory. + ValueError: Could not find the extracted file. """ # The .cmd files are intended to be included directly by Make, so they # escape the pound sign '#', either as '\#' or '$(pound)' (depending on the - # kernel version). The compile_commands.json file is not interepreted + # kernel version). The compile_commands.json file is not interpreted # by Make, so this code replaces the escaped version with '#'. - prefix = command_prefix.replace('\#', '#').replace('$(pound)', '#') - - cur_dir = root_directory - expected_path = os.path.join(cur_dir, relative_path) - if not os.path.exists(expected_path): - # Try using file_directory instead. Some of the tools have a different - # style of .cmd file than the kernel. - cur_dir = file_directory - expected_path = os.path.join(cur_dir, relative_path) + prefix = cmd_prefix.replace('\#', '#').replace('$(pound)', '#') + + # Compile commands are usually run in the top level of the kbuild output + # directory + working_dir = kbuild_out_dir + + if os.path.isabs(file_path): + # This is the most common case when the kbuild output directory is + # outside the kernel source tree (e.g. cd src/linux; + # make O=/tmp/kernel-obj). In this case, the command is run in + # kbuild_out_dir, and file_path is an absolute path to the file being + # compiled. + if not os.path.exists(file_path): + raise ValueError('File %s does not exist' % file_path) + else: + # Otherwise, try to locate the file using its relative path. + # + # First, check for the file relative to kbuild_out_dir. This is the most + # common case when output directory is the same as the kernel source + # directory, or if the output directory is specified using a relative + # path (e.g. make, or make O=kernel-obj/) + expected_path = os.path.join(kbuild_out_dir, file_path) + if not os.path.exists(expected_path): - raise ValueError('File %s not in %s or %s' % - (relative_path, root_directory, file_directory)) + # Try using file_dir instead. Some of the tools have a different + # style of .cmd file than the kernel. In this case, the command is + # run in a subdirectory of the kernel source tree. The subdirectory + # will match the directory the cmd file was found in (e.g. + # /tmp/kernel-obj/tools/objtool/.weak.o.cmd contains a command + # that's run in src/linux/tools/objtool/) + + # Translate file_dir to a relative path, and use the relative path + # to locate where in the kernel source tree the command may have + # been executed. + relative_file_dir = os.path.relpath(file_dir, kbuild_out_dir) + working_dir = os.path.join(src_dir, relative_file_dir) + expected_path = os.path.join(working_dir, file_path) + + if not os.path.exists(expected_path): + # At this point, failures are often from tools/objtool/ + # and tools/lib/subcmd/ + raise ValueError('File %s not in %s or %s' % + (file_path, kbuild_out_dir, file_dir)) return { - 'directory': cur_dir, - 'file': relative_path, - 'command': prefix + relative_path, + 'directory': working_dir, + 'file': file_path, + 'command': prefix + file_path, } def main(): """Walks through the directory and finds and parses .cmd files.""" - log_level, directory, output = parse_arguments() + log_level, src_dir, kbuild_out_dir, out_file = parse_arguments() level = getattr(logging, log_level) logging.basicConfig(format='%(levelname)s: %(message)s', level=level) @@ -118,7 +159,7 @@ def main(): line_matcher = re.compile(_LINE_PATTERN) compile_commands = [] - for dirpath, _, filenames in os.walk(directory): + for dirpath, _, filenames in os.walk(kbuild_out_dir): for filename in filenames: if not filename_matcher.match(filename): continue @@ -131,14 +172,15 @@ def main(): continue try: - entry = process_line(directory, dirpath, - result.group(1), result.group(2)) + entry = process_line(src_dir, kbuild_out_dir, + dirpath, result.group(1), + result.group(2)) compile_commands.append(entry) except ValueError as err: logging.info('Could not add line from %s: %s', filepath, err) - with open(output, 'wt') as f: + with open(out_file, 'wt') as f: json.dump(compile_commands, f, indent=2, sort_keys=True) count = len(compile_commands) -- 2.28.0.163.g6104cc2f0b6-goog
| |