1#!/usr/bin/env python3
2#
3# ===- clang-tidy-diff.py - ClangTidy Diff Checker -----------*- python -*--===#
4#
5# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
6# See https://llvm.org/LICENSE.txt for license information.
7# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
8#
9# ===-----------------------------------------------------------------------===#
10
11r"""
12ClangTidy Diff Checker
13======================
14
15This script reads input from a unified diff, runs clang-tidy on all changed
16files and outputs clang-tidy warnings in changed lines only. This is useful to
17detect clang-tidy regressions in the lines touched by a specific patch.
18Example usage for git/svn users:
19
20  git diff -U0 HEAD^ | clang-tidy-diff.py -p1
21  svn diff --diff-cmd=diff -x-U0 | \
22      clang-tidy-diff.py -fix -checks=-*,modernize-use-override
23
24"""
25
26import argparse
27import glob
28import json
29import multiprocessing
30import os
31import re
32import shutil
33import subprocess
34import sys
35import tempfile
36import threading
37import traceback
38
39try:
40    import yaml
41except ImportError:
42    yaml = None
43
44is_py2 = sys.version[0] == "2"
45
46if is_py2:
47    import Queue as queue
48else:
49    import queue as queue
50
51
52def run_tidy(task_queue, lock, timeout, failed_files):
53    watchdog = None
54    while True:
55        command = task_queue.get()
56        try:
57            proc = subprocess.Popen(
58                command, stdout=subprocess.PIPE, stderr=subprocess.PIPE
59            )
60
61            if timeout is not None:
62                watchdog = threading.Timer(timeout, proc.kill)
63                watchdog.start()
64
65            stdout, stderr = proc.communicate()
66            if proc.returncode != 0:
67                if proc.returncode < 0:
68                    msg = "Terminated by signal %d : %s\n" % (
69                        -proc.returncode,
70                        " ".join(command),
71                    )
72                    stderr += msg.encode("utf-8")
73                failed_files.append(command)
74
75            with lock:
76                sys.stdout.write(stdout.decode("utf-8") + "\n")
77                sys.stdout.flush()
78                if stderr:
79                    sys.stderr.write(stderr.decode("utf-8") + "\n")
80                    sys.stderr.flush()
81        except Exception as e:
82            with lock:
83                sys.stderr.write("Failed: " + str(e) + ": ".join(command) + "\n")
84        finally:
85            with lock:
86                if not (timeout is None or watchdog is None):
87                    if not watchdog.is_alive():
88                        sys.stderr.write(
89                            "Terminated by timeout: " + " ".join(command) + "\n"
90                        )
91                    watchdog.cancel()
92            task_queue.task_done()
93
94
95def start_workers(max_tasks, tidy_caller, arguments):
96    for _ in range(max_tasks):
97        t = threading.Thread(target=tidy_caller, args=arguments)
98        t.daemon = True
99        t.start()
100
101
102def merge_replacement_files(tmpdir, mergefile):
103    """Merge all replacement files in a directory into a single file"""
104    # The fixes suggested by clang-tidy >= 4.0.0 are given under
105    # the top level key 'Diagnostics' in the output yaml files
106    mergekey = "Diagnostics"
107    merged = []
108    for replacefile in glob.iglob(os.path.join(tmpdir, "*.yaml")):
109        content = yaml.safe_load(open(replacefile, "r"))
110        if not content:
111            continue  # Skip empty files.
112        merged.extend(content.get(mergekey, []))
113
114    if merged:
115        # MainSourceFile: The key is required by the definition inside
116        # include/clang/Tooling/ReplacementsYaml.h, but the value
117        # is actually never used inside clang-apply-replacements,
118        # so we set it to '' here.
119        output = {"MainSourceFile": "", mergekey: merged}
120        with open(mergefile, "w") as out:
121            yaml.safe_dump(output, out)
122    else:
123        # Empty the file:
124        open(mergefile, "w").close()
125
126
127def main():
128    parser = argparse.ArgumentParser(
129        description="Run clang-tidy against changed files, and "
130        "output diagnostics only for modified "
131        "lines."
132    )
133    parser.add_argument(
134        "-clang-tidy-binary",
135        metavar="PATH",
136        default="clang-tidy",
137        help="path to clang-tidy binary",
138    )
139    parser.add_argument(
140        "-p",
141        metavar="NUM",
142        default=0,
143        help="strip the smallest prefix containing P slashes",
144    )
145    parser.add_argument(
146        "-regex",
147        metavar="PATTERN",
148        default=None,
149        help="custom pattern selecting file paths to check "
150        "(case sensitive, overrides -iregex)",
151    )
152    parser.add_argument(
153        "-iregex",
154        metavar="PATTERN",
155        default=r".*\.(cpp|cc|c\+\+|cxx|c|cl|h|hpp|m|mm|inc)",
156        help="custom pattern selecting file paths to check "
157        "(case insensitive, overridden by -regex)",
158    )
159    parser.add_argument(
160        "-j",
161        type=int,
162        default=1,
163        help="number of tidy instances to be run in parallel.",
164    )
165    parser.add_argument(
166        "-timeout", type=int, default=None, help="timeout per each file in seconds."
167    )
168    parser.add_argument(
169        "-fix", action="store_true", default=False, help="apply suggested fixes"
170    )
171    parser.add_argument(
172        "-checks",
173        help="checks filter, when not specified, use clang-tidy " "default",
174        default="",
175    )
176    parser.add_argument(
177        "-config-file",
178        dest="config_file",
179        help="Specify the path of .clang-tidy or custom config file",
180        default="",
181    )
182    parser.add_argument("-use-color", action="store_true", help="Use colors in output")
183    parser.add_argument(
184        "-path", dest="build_path", help="Path used to read a compile command database."
185    )
186    if yaml:
187        parser.add_argument(
188            "-export-fixes",
189            metavar="FILE_OR_DIRECTORY",
190            dest="export_fixes",
191            help="A directory or a yaml file to store suggested fixes in, "
192            "which can be applied with clang-apply-replacements. If the "
193            "parameter is a directory, the fixes of each compilation unit are "
194            "stored in individual yaml files in the directory.",
195        )
196    else:
197        parser.add_argument(
198            "-export-fixes",
199            metavar="DIRECTORY",
200            dest="export_fixes",
201            help="A directory to store suggested fixes in, which can be applied "
202            "with clang-apply-replacements. The fixes of each compilation unit are "
203            "stored in individual yaml files in the directory.",
204        )
205    parser.add_argument(
206        "-extra-arg",
207        dest="extra_arg",
208        action="append",
209        default=[],
210        help="Additional argument to append to the compiler " "command line.",
211    )
212    parser.add_argument(
213        "-extra-arg-before",
214        dest="extra_arg_before",
215        action="append",
216        default=[],
217        help="Additional argument to prepend to the compiler " "command line.",
218    )
219    parser.add_argument(
220        "-quiet",
221        action="store_true",
222        default=False,
223        help="Run clang-tidy in quiet mode",
224    )
225    parser.add_argument(
226        "-load",
227        dest="plugins",
228        action="append",
229        default=[],
230        help="Load the specified plugin in clang-tidy.",
231    )
232
233    clang_tidy_args = []
234    argv = sys.argv[1:]
235    if "--" in argv:
236        clang_tidy_args.extend(argv[argv.index("--") :])
237        argv = argv[: argv.index("--")]
238
239    args = parser.parse_args(argv)
240
241    # Extract changed lines for each file.
242    filename = None
243    lines_by_file = {}
244    for line in sys.stdin:
245        match = re.search('^\+\+\+\ "?(.*?/){%s}([^ \t\n"]*)' % args.p, line)
246        if match:
247            filename = match.group(2)
248        if filename is None:
249            continue
250
251        if args.regex is not None:
252            if not re.match("^%s$" % args.regex, filename):
253                continue
254        else:
255            if not re.match("^%s$" % args.iregex, filename, re.IGNORECASE):
256                continue
257
258        match = re.search("^@@.*\+(\d+)(,(\d+))?", line)
259        if match:
260            start_line = int(match.group(1))
261            line_count = 1
262            if match.group(3):
263                line_count = int(match.group(3))
264            if line_count == 0:
265                continue
266            end_line = start_line + line_count - 1
267            lines_by_file.setdefault(filename, []).append([start_line, end_line])
268
269    if not any(lines_by_file):
270        print("No relevant changes found.")
271        sys.exit(0)
272
273    max_task_count = args.j
274    if max_task_count == 0:
275        max_task_count = multiprocessing.cpu_count()
276    max_task_count = min(len(lines_by_file), max_task_count)
277
278    combine_fixes = False
279    export_fixes_dir = None
280    delete_fixes_dir = False
281    if args.export_fixes is not None:
282        # if a directory is given, create it if it does not exist
283        if args.export_fixes.endswith(os.path.sep) and not os.path.isdir(
284            args.export_fixes
285        ):
286            os.makedirs(args.export_fixes)
287
288        if not os.path.isdir(args.export_fixes):
289            if not yaml:
290                raise RuntimeError(
291                    "Cannot combine fixes in one yaml file. Either install PyYAML or specify an output directory."
292                )
293
294            combine_fixes = True
295
296        if os.path.isdir(args.export_fixes):
297            export_fixes_dir = args.export_fixes
298
299    if combine_fixes:
300        export_fixes_dir = tempfile.mkdtemp()
301        delete_fixes_dir = True
302
303    # Tasks for clang-tidy.
304    task_queue = queue.Queue(max_task_count)
305    # A lock for console output.
306    lock = threading.Lock()
307
308    # List of files with a non-zero return code.
309    failed_files = []
310
311    # Run a pool of clang-tidy workers.
312    start_workers(
313        max_task_count, run_tidy, (task_queue, lock, args.timeout, failed_files)
314    )
315
316    # Form the common args list.
317    common_clang_tidy_args = []
318    if args.fix:
319        common_clang_tidy_args.append("-fix")
320    if args.checks != "":
321        common_clang_tidy_args.append("-checks=" + args.checks)
322    if args.config_file != "":
323        common_clang_tidy_args.append("-config-file=" + args.config_file)
324    if args.quiet:
325        common_clang_tidy_args.append("-quiet")
326    if args.build_path is not None:
327        common_clang_tidy_args.append("-p=%s" % args.build_path)
328    if args.use_color:
329        common_clang_tidy_args.append("--use-color")
330    for arg in args.extra_arg:
331        common_clang_tidy_args.append("-extra-arg=%s" % arg)
332    for arg in args.extra_arg_before:
333        common_clang_tidy_args.append("-extra-arg-before=%s" % arg)
334    for plugin in args.plugins:
335        common_clang_tidy_args.append("-load=%s" % plugin)
336
337    for name in lines_by_file:
338        line_filter_json = json.dumps(
339            [{"name": name, "lines": lines_by_file[name]}], separators=(",", ":")
340        )
341
342        # Run clang-tidy on files containing changes.
343        command = [args.clang_tidy_binary]
344        command.append("-line-filter=" + line_filter_json)
345        if args.export_fixes is not None:
346            # Get a temporary file. We immediately close the handle so clang-tidy can
347            # overwrite it.
348            (handle, tmp_name) = tempfile.mkstemp(suffix=".yaml", dir=export_fixes_dir)
349            os.close(handle)
350            command.append("-export-fixes=" + tmp_name)
351        command.extend(common_clang_tidy_args)
352        command.append(name)
353        command.extend(clang_tidy_args)
354
355        task_queue.put(command)
356
357    # Application return code
358    return_code = 0
359
360    # Wait for all threads to be done.
361    task_queue.join()
362    # Application return code
363    return_code = 0
364    if failed_files:
365        return_code = 1
366
367    if combine_fixes:
368        print("Writing fixes to " + args.export_fixes + " ...")
369        try:
370            merge_replacement_files(export_fixes_dir, args.export_fixes)
371        except:
372            sys.stderr.write("Error exporting fixes.\n")
373            traceback.print_exc()
374            return_code = 1
375
376    if delete_fixes_dir:
377        shutil.rmtree(export_fixes_dir)
378    sys.exit(return_code)
379
380
381if __name__ == "__main__":
382    main()
383