File: code-format-helper.py

package info (click to toggle)
llvm-toolchain-19 1%3A19.1.4-1~deb12u1
  • links: PTS, VCS
  • area: main
  • in suites: bookworm
  • size: 1,998,488 kB
  • sloc: cpp: 6,951,470; ansic: 1,486,052; asm: 913,550; python: 232,020; f90: 80,126; objc: 75,349; lisp: 37,276; pascal: 16,990; sh: 9,935; ml: 5,058; perl: 4,724; awk: 3,523; makefile: 3,164; javascript: 2,504; xml: 892; fortran: 664; cs: 573
file content (405 lines) | stat: -rwxr-xr-x 13,336 bytes parent folder | download | duplicates (3)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
#!/usr/bin/env python3
#
# ====- code-format-helper, runs code formatters from the ci or in a hook --*- python -*--==#
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
# ==--------------------------------------------------------------------------------------==#

import argparse
import os
import subprocess
import sys
from typing import List, Optional

"""
This script is run by GitHub actions to ensure that the code in PR's conform to
the coding style of LLVM. It can also be installed as a pre-commit git hook to
check the coding style before submitting it. The canonical source of this script
is in the LLVM source tree under llvm/utils/git.

For C/C++ code it uses clang-format and for Python code it uses darker (which
in turn invokes black).

You can learn more about the LLVM coding style on llvm.org:
https://llvm.org/docs/CodingStandards.html

You can install this script as a git hook by symlinking it to the .git/hooks
directory:

ln -s $(pwd)/llvm/utils/git/code-format-helper.py .git/hooks/pre-commit

You can control the exact path to clang-format or darker with the following
environment variables: $CLANG_FORMAT_PATH and $DARKER_FORMAT_PATH.
"""


class FormatArgs:
    start_rev: str = None
    end_rev: str = None
    repo: str = None
    changed_files: List[str] = []
    token: str = None
    verbose: bool = True
    issue_number: int = 0
    write_comment_to_file: bool = False

    def __init__(self, args: argparse.Namespace = None) -> None:
        if not args is None:
            self.start_rev = args.start_rev
            self.end_rev = args.end_rev
            self.repo = args.repo
            self.token = args.token
            self.changed_files = args.changed_files
            self.issue_number = args.issue_number
            self.write_comment_to_file = args.write_comment_to_file


class FormatHelper:
    COMMENT_TAG = "<!--LLVM CODE FORMAT COMMENT: {fmt}-->"
    name: str
    friendly_name: str
    comment: dict = None

    @property
    def comment_tag(self) -> str:
        return self.COMMENT_TAG.replace("fmt", self.name)

    @property
    def instructions(self) -> str:
        raise NotImplementedError()

    def has_tool(self) -> bool:
        raise NotImplementedError()

    def format_run(self, changed_files: List[str], args: FormatArgs) -> Optional[str]:
        raise NotImplementedError()

    def pr_comment_text_for_diff(self, diff: str) -> str:
        return f"""
:warning: {self.friendly_name}, {self.name} found issues in your code. :warning:

<details>
<summary>
You can test this locally with the following command:
</summary>

``````````bash
{self.instructions}
``````````

</details>

<details>
<summary>
View the diff from {self.name} here.
</summary>

``````````diff
{diff}
``````````

</details>
"""

    # TODO: any type should be replaced with the correct github type, but it requires refactoring to
    # not require the github module to be installed everywhere.
    def find_comment(self, pr: any) -> any:
        for comment in pr.as_issue().get_comments():
            if self.comment_tag in comment.body:
                return comment
        return None

    def update_pr(self, comment_text: str, args: FormatArgs, create_new: bool) -> None:
        import github
        from github import IssueComment, PullRequest

        repo = github.Github(args.token).get_repo(args.repo)
        pr = repo.get_issue(args.issue_number).as_pull_request()

        comment_text = self.comment_tag + "\n\n" + comment_text

        existing_comment = self.find_comment(pr)

        if args.write_comment_to_file:
            if create_new or existing_comment:
                self.comment = {"body": comment_text}
            if existing_comment:
                self.comment["id"] = existing_comment.id
            return

        if existing_comment:
            existing_comment.edit(comment_text)
        elif create_new:
            pr.as_issue().create_comment(comment_text)

    def run(self, changed_files: List[str], args: FormatArgs) -> bool:
        changed_files = [arg for arg in changed_files if "third-party" not in arg]
        diff = self.format_run(changed_files, args)
        should_update_gh = args.token is not None and args.repo is not None

        if diff is None:
            if should_update_gh:
                comment_text = (
                    ":white_check_mark: With the latest revision "
                    f"this PR passed the {self.friendly_name}."
                )
                self.update_pr(comment_text, args, create_new=False)
            return True
        elif len(diff) > 0:
            if should_update_gh:
                comment_text = self.pr_comment_text_for_diff(diff)
                self.update_pr(comment_text, args, create_new=True)
            else:
                print(
                    f"Warning: {self.friendly_name}, {self.name} detected "
                    "some issues with your code formatting..."
                )
            return False
        else:
            # The formatter failed but didn't output a diff (e.g. some sort of
            # infrastructure failure).
            comment_text = (
                f":warning: The {self.friendly_name} failed without printing "
                "a diff. Check the logs for stderr output. :warning:"
            )
            self.update_pr(comment_text, args, create_new=False)
            return False


class ClangFormatHelper(FormatHelper):
    name = "clang-format"
    friendly_name = "C/C++ code formatter"

    @property
    def instructions(self) -> str:
        return " ".join(self.cf_cmd)

    def should_include_extensionless_file(self, path: str) -> bool:
        return path.startswith("libcxx/include")

    def filter_changed_files(self, changed_files: List[str]) -> List[str]:
        filtered_files = []
        for path in changed_files:
            _, ext = os.path.splitext(path)
            if ext in (".cpp", ".c", ".h", ".hpp", ".hxx", ".cxx", ".inc", ".cppm"):
                filtered_files.append(path)
            elif ext == "" and self.should_include_extensionless_file(path):
                filtered_files.append(path)
        return filtered_files

    @property
    def clang_fmt_path(self) -> str:
        if "CLANG_FORMAT_PATH" in os.environ:
            return os.environ["CLANG_FORMAT_PATH"]
        return "git-clang-format"

    def has_tool(self) -> bool:
        cmd = [self.clang_fmt_path, "-h"]
        proc = None
        try:
            proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        except:
            return False
        return proc.returncode == 0

    def format_run(self, changed_files: List[str], args: FormatArgs) -> Optional[str]:
        cpp_files = self.filter_changed_files(changed_files)
        if not cpp_files:
            return None

        cf_cmd = [self.clang_fmt_path, "--diff"]

        if args.start_rev and args.end_rev:
            cf_cmd.append(args.start_rev)
            cf_cmd.append(args.end_rev)

        # Gather the extension of all modified files and pass them explicitly to git-clang-format.
        # This prevents git-clang-format from applying its own filtering rules on top of ours.
        extensions = set()
        for file in cpp_files:
            _, ext = os.path.splitext(file)
            extensions.add(
                ext.strip(".")
            )  # Exclude periods since git-clang-format takes extensions without them
        cf_cmd.append("--extensions")
        cf_cmd.append(",".join(extensions))

        cf_cmd.append("--")
        cf_cmd += cpp_files

        if args.verbose:
            print(f"Running: {' '.join(cf_cmd)}")
        self.cf_cmd = cf_cmd
        proc = subprocess.run(cf_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        sys.stdout.write(proc.stderr.decode("utf-8"))

        if proc.returncode != 0:
            # formatting needed, or the command otherwise failed
            if args.verbose:
                print(f"error: {self.name} exited with code {proc.returncode}")
                # Print the diff in the log so that it is viewable there
                print(proc.stdout.decode("utf-8"))
            return proc.stdout.decode("utf-8")
        else:
            return None


class DarkerFormatHelper(FormatHelper):
    name = "darker"
    friendly_name = "Python code formatter"

    @property
    def instructions(self) -> str:
        return " ".join(self.darker_cmd)

    def filter_changed_files(self, changed_files: List[str]) -> List[str]:
        filtered_files = []
        for path in changed_files:
            name, ext = os.path.splitext(path)
            if ext == ".py":
                filtered_files.append(path)

        return filtered_files

    @property
    def darker_fmt_path(self) -> str:
        if "DARKER_FORMAT_PATH" in os.environ:
            return os.environ["DARKER_FORMAT_PATH"]
        return "darker"

    def has_tool(self) -> bool:
        cmd = [self.darker_fmt_path, "--version"]
        proc = None
        try:
            proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        except:
            return False
        return proc.returncode == 0

    def format_run(self, changed_files: List[str], args: FormatArgs) -> Optional[str]:
        py_files = self.filter_changed_files(changed_files)
        if not py_files:
            return None
        darker_cmd = [
            self.darker_fmt_path,
            "--check",
            "--diff",
        ]
        if args.start_rev and args.end_rev:
            darker_cmd += ["-r", f"{args.start_rev}...{args.end_rev}"]
        darker_cmd += py_files
        if args.verbose:
            print(f"Running: {' '.join(darker_cmd)}")
        self.darker_cmd = darker_cmd
        proc = subprocess.run(
            darker_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE
        )
        if args.verbose:
            sys.stdout.write(proc.stderr.decode("utf-8"))

        if proc.returncode != 0:
            # formatting needed, or the command otherwise failed
            if args.verbose:
                print(f"error: {self.name} exited with code {proc.returncode}")
                # Print the diff in the log so that it is viewable there
                print(proc.stdout.decode("utf-8"))
            return proc.stdout.decode("utf-8")
        else:
            sys.stdout.write(proc.stdout.decode("utf-8"))
            return None


ALL_FORMATTERS = (DarkerFormatHelper(), ClangFormatHelper())


def hook_main():
    # fill out args
    args = FormatArgs()
    args.verbose = False

    # find the changed files
    cmd = ["git", "diff", "--cached", "--name-only", "--diff-filter=d"]
    proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    output = proc.stdout.decode("utf-8")
    for line in output.splitlines():
        args.changed_files.append(line)

    failed_fmts = []
    for fmt in ALL_FORMATTERS:
        if fmt.has_tool():
            if not fmt.run(args.changed_files, args):
                failed_fmts.append(fmt.name)
            if fmt.comment:
                comments.append(fmt.comment)
        else:
            print(f"Couldn't find {fmt.name}, can't check " + fmt.friendly_name.lower())

    if len(failed_fmts) > 0:
        sys.exit(1)

    sys.exit(0)


if __name__ == "__main__":
    script_path = os.path.abspath(__file__)
    if ".git/hooks" in script_path:
        hook_main()
        sys.exit(0)

    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--token", type=str, required=True, help="GitHub authentiation token"
    )
    parser.add_argument(
        "--repo",
        type=str,
        default=os.getenv("GITHUB_REPOSITORY", "llvm/llvm-project"),
        help="The GitHub repository that we are working with in the form of <owner>/<repo> (e.g. llvm/llvm-project)",
    )
    parser.add_argument("--issue-number", type=int, required=True)
    parser.add_argument(
        "--start-rev",
        type=str,
        required=True,
        help="Compute changes from this revision.",
    )
    parser.add_argument(
        "--end-rev", type=str, required=True, help="Compute changes to this revision"
    )
    parser.add_argument(
        "--changed-files",
        type=str,
        help="Comma separated list of files that has been changed",
    )
    parser.add_argument(
        "--write-comment-to-file",
        action="store_true",
        help="Don't post comments on the PR, instead write the comments and metadata a file called 'comment'",
    )

    args = FormatArgs(parser.parse_args())

    changed_files = []
    if args.changed_files:
        changed_files = args.changed_files.split(",")

    failed_formatters = []
    comments = []
    for fmt in ALL_FORMATTERS:
        if not fmt.run(changed_files, args):
            failed_formatters.append(fmt.name)
        if fmt.comment:
            comments.append(fmt.comment)

    if len(comments):
        with open("comments", "w") as f:
            import json

            json.dump(comments, f)

    if len(failed_formatters) > 0:
        print(f"error: some formatters failed: {' '.join(failed_formatters)}")
        sys.exit(1)