File: process-templates.py

package info (click to toggle)
libvirt 11.6.0-1
  • links: PTS, VCS
  • area: main
  • in suites: experimental
  • size: 205,260 kB
  • sloc: ansic: 533,672; xml: 310,954; python: 11,881; perl: 2,629; sh: 2,185; makefile: 448; javascript: 126; cpp: 22
file content (327 lines) | stat: -rwxr-xr-x 9,133 bytes parent folder | download | duplicates (3)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
#!/usr/bin/env python3

import argparse
import subprocess
import sys

from pathlib import Path


COMMON_DESCRIPTION = """
libvirt exposes a long-term stable API that can be used to interact with
various hypervisors. Its architecture is highly modular, with most features
implemented as optional drivers. It can be used from C as well as several
other programming languages, and it forms the basis of virtualization
solutions tailored for a range of use cases.
"""


def read_file(path):
    with open(path) as f:
        return f.read()


def write_file(path, output):
    with open(path, "w") as f:
        f.write(output)


# Parse a file that looks like
#
#   FOO = foo1 foo2 foo3
#   BAR =
#   BAZ = baz1
#
# into a dictionary that looks like
#
#   {
#     "${FOO}": ["foo1", "foo2", "foo3"],
#     "${BAR}": [],
#     "${BAZ}": ["baz1"],
#   }
#
def load_values(path):
    values = {}

    with open(path) as f:
        lineno = 0
        for line in f:
            lineno += 1
            line = line.strip()

            # Skip empty lines and comments
            if line == "" or line[0] == "#":
                continue

            parts = line.split(" ")

            if len(parts) < 2 or parts[1] != "=":
                print(f"{path}:{lineno}: Invalid syntax")
                sys.exit(1)

            # Use "${FOO}" instead of "FOO" as the dictionary key.
            # This makes things more convenient later
            key = "${" + parts[0] + "}"
            value = parts[2:]

            values[key] = value

    return values


# Parse a file that looks like
#
#   #BEGIN FOO
#   foo() { touch foo }
#   #END FOO
#
#   #BEGIN BAR
#   bar() { rm -f bar }
#   #END BAR
#
# into a dictionary that looks like
#
#   {
#     "#FOO#": "foo() { touch foo }",
#     "#BAR#": "bar() { rm -f bar }",
#   }
#
def load_snippets(path):
    snippets = {}

    with open(path) as f:
        lineno = 0
        current = "NONE"
        snippet = []

        for line in f:
            lineno += 1

            # Only strip trailing whitespace to preserve indentation
            line = line.rstrip()

            # Currently not inside a snippet
            if current == "NONE":

                # Skip empty lines
                if line == "":
                    continue

                # Start of a new snippet
                if line.startswith("#BEGIN "):
                    current = "#" + line[len("#BEGIN "):] + "#"
                    continue

                # The only thing accepted outside of a snippet is the
                # start of a snippet
                print(f"{path}:{lineno}: Invalid syntax")
                sys.exit(1)

            # Currently inside a snippet
            else:

                # End of the current snippet
                if line.startswith("#END "):
                    name = "#" + line[len("#END "):] + "#"

                    # Prevent mismatched BEGIN/END
                    if name != current:
                        print(f"{path}:{lineno}: Expected {current}, got {name}")
                        sys.exit(1)

                    # Save the current snippet and start fresh
                    snippets[current] = "\n".join(snippet)
                    current = "NONE"
                    snippet = []
                    continue

                # The rest of the snippet is taken verbatim
                snippet.append(line)

        # Final sanity check
        if len(snippet) != 0 or current != "NONE":
            print(f"{path}: Last snippet was not terminated")
            sys.exit(1)

    return snippets


def common_description():
    desc = []

    for line in COMMON_DESCRIPTION.split("\n"):
        if line == "":
            continue
        desc.append(" " + line)

    return "\n".join(desc).lstrip()


def process_control(path, arches):
    output = read_file(path)

    for arch in arches:
        output = output.replace(arch, " ".join(arches[arch]))

    output = output.replace("@COMMON_DESCRIPTION@", common_description())

    return output


def process_maintscript(path, snippets):
    output = read_file(path)

    for snippet in snippets:
        output = output.replace(snippet, snippets[snippet])

    return output


def process_debhelper(path, arches, mode, arch, os):
    output = []

    with open(path) as f:
        lineno = 0
        for line in f:
            lineno += 1
            line = line.strip()

            # Empty lines and lines that don't start with [cond] are
            # included in the output verbatim
            if line == "" or line[0] != "[":
                output.append(line)
                continue

            parts = line[1:].split("]", maxsplit=1)

            if len(parts) < 2:
                print(f"{path}:{lineno}: Invalid syntax")
                sys.exit(1)

            # The line looked like
            #
            #   [cond] file
            #
            cond = parts[0].strip()
            file = parts[1].strip()

            # In verify mode, strip the condition and output the rest.
            # Running wrap-and-sort against this output (see below)
            # guarantees that the input follows the requirements too
            if mode == "verify":
                output.append(file)
                continue

            # Handle lines that look like
            #
            #   [linux-any] file
            #
            if cond.endswith("-any"):
                if cond == os + "-any":
                    output.append(file)
                continue

            if cond not in arches:
                print(f"{path}:{lineno}: Unknown architecture group '{cond}'")
                sys.exit(1)

            # Only output the line if we the architecture we're building on
            # is one of those listed in cond. cond itself will be stripped
            if arch in arches[cond]:
                output.append(file)

    output.append("")
    return "\n".join(output)


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--mode", choices=["generate", "build", "verify"],
                                  default="generate")
    parser.add_argument("--arch")
    parser.add_argument("--os")
    args = parser.parse_args()
    mode = args.mode
    arch = args.arch
    os = args.os

    if mode == "build" and (arch is None or os is None):
        print("--arch and --os are required for --mode=build")
        sys.exit(1)

    maintscript_exts = [
        ".postinst",
        ".postrm",
        ".preinst",
        ".prerm",
    ]
    debhelper_exts = [
        ".install",
    ]
    template_ext = ".in"
    debian_dir = Path("debian")
    arches_file = Path(debian_dir, "arches.mk")
    snippets_file = Path(debian_dir, "snippets.sh")

    arches = load_values(arches_file)
    snippets = load_snippets(snippets_file)

    for infile in sorted(debian_dir.glob("*")):
        infile = Path(infile)

        # Only process templates
        if infile.suffix != template_ext:
            continue

        outfile = Path(debian_dir, infile.stem)

        # Generate mode is for maintainers, and is used to keep
        # debian/control in sync with its template.
        # All other files are ignored
        if mode == "generate" and outfile.name != "control":
            continue

        print(f"  GEN {outfile}")

        # When building the package, debian/control should already be
        # in sync with its template. To confirm that is the case,
        # save the contents of the output file before regenerating it
        if mode in ["build", "verify"] and outfile.name == "control":
            old_output = read_file(outfile)

        if outfile.name == "control":
            output = process_control(infile, arches)
        elif outfile.suffix in maintscript_exts:
            output = process_maintscript(infile, snippets)
        elif outfile.suffix in debhelper_exts:
            output = process_debhelper(infile, arches, mode, arch, os)
        else:
            print(f"Unknown file type {outfile.suffix}")
            sys.exit(1)

        write_file(outfile, output)

        # When building the package, regenerating debian/control
        # should be a no-op. If that's not the case, it means that
        # the file and its template have gone out of sync, and we
        # don't know which one should be used.
        # Abort the build and let the user fix things
        if mode in ["build", "verify"] and outfile.name == "control":
            if output != old_output:
                print(f"{outfile}: Needs to be regenerated from template")
                sys.exit(1)

    # In verify mode only, check that things are pretty
    if mode == "verify":
        print("  CHK wrap-and-sort")
        wrap_and_sort = subprocess.run(["wrap-and-sort", "-ast", "--dry-run"],
                                       capture_output=True, text=True)
        if wrap_and_sort.returncode != 0 or wrap_and_sort.stdout != "":
            print("stdout:")
            print(wrap_and_sort.stdout.strip())
            print(f"rc: {wrap_and_sort.returncode}\n")
            sys.exit(1)


if __name__ == "__main__":
    main()