File: quick-start-build.py

package info (click to toggle)
ispc 1.28.2-1
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid
  • size: 97,620 kB
  • sloc: cpp: 77,067; python: 8,303; yacc: 3,337; lex: 1,126; ansic: 631; sh: 475; makefile: 17
file content (530 lines) | stat: -rwxr-xr-x 19,794 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
#!/usr/bin/env python3

# Copyright (c) 2025, Intel Corporation
#
# SPDX-License-Identifier: BSD-3-Clause

import os
import sys
import platform
import subprocess # nosec
import json
import shutil
from pathlib import Path
import re
import tarfile
import argparse
from urllib.request import urlopen, Request
from urllib.error import HTTPError
from multiprocessing import cpu_count


def get_llvm_asset(llvm_version, os_name, arch):
    """
    Find and retrieve LLVM release asset information based on the current system architecture
    and specified LLVM version from the ispc.dependencies GitHub repository.

    This function performs the following steps:
    1. Detects and normalizes the current OS and architecture to the form that
       is used in ispc.dependencies assets naming
    2. Queries GitHub API to find matching LLVM releases
    3. Locates the appropriate asset for the current platform
    4. Returns asset information for downloading

    Args:
        llvm_version (str): The desired LLVM version (e.g., "18", "17")
        os_name (str): The current operating system (e.g., "Linux", "Darwin", "Windows")
        arch (str): The current architecture (e.g., "x86_64", "aarch64")

    Returns:
        tuple: A tuple containing three elements:
            - asset_name (str): Name of the asset file
            - asset_url (str): Direct download URL for the asset
            - version (str): Extracted version number
            Or (None, None, None) if no matching asset is found or on API error

    Environment Requirements:
        - Supported operating systems: Linux (Ubuntu 22.04), macOS
        - Supported architectures: x86_64, aarch64 (Linux), arm64 (macOS)

    Example:
        >>> asset_name, asset_url, version = get_llvm_asset("18")
        Found release: llvm-18.0.0
        >>> print(asset_name)
        llvm-18.0.0-ubuntu22.04-Release-Assert-x86_64.tar.xz

    Notes:
        - The function uses GitHub's public API and may be subject to rate limiting
        - For Linux arm64, the asset name includes 'aarch64'
        - For macOS arm64, the asset name does not include architecture
        - The function excludes LTO (Link Time Optimization) variants of assets
    """
    # Normalize OS names
    os_map = {
        'Linux': 'ubuntu22.04',
        'Darwin': 'macos',
        'Windows': 'win'
    }

    if os_name not in os_map:
        raise RuntimeError(f"Unsupported OS: {os_name}")

    os_name = os_map[os_name]

    # Normalize architecture names
    if arch == 'x86_64' or arch == 'AMD64':
        arch = ''
    elif arch == 'aarch64':
        arch = 'aarch64'  # linux tarball contains aarch64 after OS
    elif arch == 'arm64':
        arch = ''  # macOS specific, tarballs don't contain arm in their names
    else:
        raise RuntimeError(f"Unsupported architecture: {arch}")

    # Fetch GitHub releases
    headers = {'User-Agent': 'Python'}
    try:
        req = Request("https://api.github.com/repos/ispc/ispc.dependencies/releases", headers=headers)
        with urlopen(req) as response: # nosec
            releases_json = json.loads(response.read())
    except HTTPError as e:
        if "rate limit exceeded" in str(e):
            print("GitHub API rate limit exceeded.")
            return None, None, None
        raise

    # Find matching release
    matching_release = None
    for release in releases_json:
        if release['tag_name'].startswith(f"llvm-{llvm_version}."):
            matching_release = release['tag_name']
            break

    if not matching_release:
        print(f"No matching release found for llvm-{llvm_version}.*")
        return None, None, None

    print(f"Found release: {matching_release}")
    version = matching_release[5:].split('-')[0]  # Remove 'llvm-' prefix and everything after first '-'

    # Fetch assets for the matching release
    try:
        req = Request(f"https://api.github.com/repos/ispc/ispc.dependencies/releases/tags/{matching_release}", headers=headers)
        with urlopen(req) as response: # nosec
            assets_json = json.loads(response.read())
    except HTTPError as e:
        if "rate limit exceeded" in str(e):
            print("GitHub API rate limit exceeded.")
            return None, None, None
        raise

    # Find matching asset
    asset_pattern = f"llvm-{llvm_version}.*-{os_name}{arch}-Release.*Asserts-.*\\.tar\\.xz"
    if os_name == "win":
        asset_pattern = f"llvm-{llvm_version}.*-{os_name}.*-Release.*Asserts-.*\\.tar\\.7z"
    for asset in assets_json['assets']:
        if re.match(asset_pattern, asset['name']) and 'lto' not in asset['name']:
            return asset['name'], asset['browser_download_url'], version

    print(f"No matching assets found for release {matching_release} and pattern {asset_pattern}")
    return None, None, None


def download_file(url, filename):
    """Download a file from a URL to a local destination with progress indication.

    This function downloads a file from the specified URL and saves it locally,
    displaying a progress bar when the content length is available. It handles
    both cases where content length is known and unknown.

    Args:
        url (str): The URL of the file to download. Must be a valid URL that points
            to the target file. Supports HTTP and HTTPS protocols.
        filename (str): The local path where the downloaded file will be saved.
            If the path doesn't exist, intermediary directories will not be created.

    Returns:
        None

    Examples:
        >>> # Download a file with known content length
        >>> download_file('https://example.com/file.zip', 'local_file.zip')
        Downloading: 23%
        Downloading: 47%
        Downloading: 98%
        Downloading: 100%

        >>> # Download a file with unknown content length
        >>> download_file('https://example.com/stream', 'local_stream.dat')

    Notes:
        - Uses a custom User-Agent header to identify the client as Python
        - For files with known size, downloads in chunks of 8192 bytes
        - Progress indication is only shown when Content-Length header is present
        - Progress updates are written to stdout with carriage return for in-place updates
        - Falls back to shutil.copyfileobj() for streams with unknown length

    Warning:
        Ensure you have write permissions in the target directory and sufficient
        disk space before downloading large files.
    """
    headers = {'User-Agent': 'Python'}
    req = Request(url, headers=headers)
    with urlopen(req) as response, open(filename, 'wb') as out_file: # nosec
        content_length = response.headers.get('Content-Length')
        if content_length:
            total_size = int(content_length)
            downloaded = 0
            chunk_size = 64 * 1024
            last_progress = -1  # Track the last printed progress

            while True:
                chunk = response.read(chunk_size)
                if not chunk:
                    break
                downloaded += len(chunk)
                out_file.write(chunk)

                # Calculate progress as an integer percentage
                progress = int((downloaded / total_size) * 100)
                if progress != last_progress:
                    print(f"\rDownloading: {progress}%", end='', flush=True)
                    last_progress = progress
            print()  # New line after finishing
        else:
            # Fallback for streams with unknown content length
            shutil.copyfileobj(response, out_file)


def extract_archive(archive_path, is_windows):
    """Extract an archive file with special handling for Windows .tar.7z files.

    This function handles archive extraction with different behavior based on the
    operating system. For Windows, it specifically handles .tar.7z files using
    py7zr, performing a two-step extraction process. For other platforms, it
    directly extracts tar archives.

    Args:
        archive_path (str or Path): Path to the archive file to extract.
            For Windows: Expected to be a .tar.7z file
            For other OS: Expected to be a .tar file
        is_windows (bool): Flag indicating if the current OS is Windows.
            True for Windows systems, False for other operating systems.

    Returns:
        None

    Examples:
        >>> # On Windows with a .tar.7z file
        >>> extract_archive('llvm-13.0.0.tar.7z', True)
        Extracting llvm-13.0.0.tar.7z
        Extracting llvm-13.0.0.tar

        >>> # On Linux/Mac with a .tar file
        >>> extract_archive('llvm-13.0.0.tar', False)
        Extracting llvm-13.0.0.tar

    Notes:
        - On Windows:
          1. First extracts the outer .7z container
          2. Then extracts the inner .tar file
          3. Requires py7zr package to be installed
        - On other platforms:
          1. Directly extracts the .tar file
        - Extracts all contents to the current working directory
        - Does not preserve original archive file

    Warning:
        - Ensure sufficient disk space for extraction
        - Extraction overwrites existing files without confirmation
        - On Windows, temporary .tar file is created during extraction
        - Current directory must be writable

    Dependencies:
        - py7zr (Windows only): Required for .7z extraction
        - tarfile: Built-in Python module for tar extraction
        - pathlib: For Path operations
    """
    if is_windows:
        try:
            import py7zr
        except ImportError:
            print("Error: py7zr is required for extracting .tar.7z files on Windows.")
            print("Please install it with 'pip install py7zr'.")
            sys.exit(1)

        # For Windows .tar.7z files
        print(f"Extracting {archive_path}")
        # First extract the .7z
        with py7zr.SevenZipFile(archive_path, mode='r') as z:
            z.extractall()
        # Then extract the resulting .tar
        archive_path = next(Path('.').glob('llvm*.tar'))

    print(f"Extracting {archive_path}")
    with tarfile.open(archive_path) as tar:
        tar.extractall()


def run_command(cmd, on_error=None, env=None):
    """Execute a shell command and handle any failures.

    Runs a subprocess with the given command and checks its return code.
    If the command fails, executes an optional error callback and exits with
    the same return code.

    Args:
        cmd (list): Command to execute as a list of strings or Path objects.
            Example: ['cmake', '--build', 'path/to/build', '--target', 'all']
        on_error (callable, optional): Function to call if command fails.
            Will be called with no arguments before program exit.
            Default: None
        env (dict, optional): Environment variables for the subprocess.
            Passed directly to subprocess.run().
            If None, uses current environment.
            Default: None

    Returns:
        subprocess.CompletedProcess: If command executes successfully

    Examples:
        >>> # Basic usage
        >>> run_command(['echo', 'test'])
        test
        <CompletedProcess(args=['echo', 'test'], returncode=0)>

        >>> # With error callback
        >>> def cleanup(): print("Cleaning up...")
        >>> run_command(['false'], on_error=cleanup)
        Command failed with exit code 1: false
        Cleaning up...
        # Exits program with code 1

        >>> # With custom environment
        >>> run_command(['printenv', 'CUSTOM'], env={'CUSTOM': 'value'})
        value

    Notes:
        - Uses subprocess.run with default settings (no shell, no output capture)
        - Command output goes directly to stdout/stderr
        - All arguments are converted to strings using str()
        - Error callback runs before program termination

    Warning:
        This function will terminate the entire program if the command fails.
        Use subprocess.run directly if you need to handle command failures differently.
    """
    sys.stdout.flush()
    result = subprocess.run(cmd, env=env) if env else subprocess.run(cmd)
    if result.returncode:
        print(f"Command failed with exit code {result.returncode}: {' '.join(map(str, cmd))}")
        if on_error:
            on_error()
        sys.exit(result.returncode)


def main():
    """Set up and build ISPC with specified LLVM version.

    This is the main entry point for the ISPC build script. It handles downloading
    and setting up LLVM dependencies, configuring the build environment, and
    running the ISPC build and test suite.

    Environment Variables:
        LLVM_HOME (str, optional): Directory for LLVM installation.
            Defaults to current working directory.
        ISPC_HOME (str, optional): Root directory of ISPC source.
            Defaults to parent directory of this script.
        ARCHIVE_URL (str, optional): Direct download URL for LLVM package.
            Required only if automatic asset detection fails.

    Command Line Args:
        llvm_version (str, optional): LLVM version to use.
            Default: "20"

    Directory Structure Created/Used:
        Working Directory (LLVM_HOME)/
            - llvm-{version}/      # LLVM installation (downloaded and extracted)
        ISPC Build Directory (ISPC_HOME)/
            - build-{version}/     # ISPC build directory
        ISPC Root Directory is determined by the script location

    Build Process:
        1. Determines system configuration (OS, architecture, CPU count)
        2. Downloads and extracts LLVM if not present
        3. Configures ISPC build with CMake
        4. Builds ISPC with parallel compilation
        5. Runs ISPC support matrix check
        6. Executes test suite

    Returns:
        None

    Examples:
        # Build with default LLVM 20
        $ python quick-start-build.py

        # Build with specific LLVM version
        $ python quick-start-build.py 17

        # Build with custom LLVM location
        $ LLVM_HOME=/path/to/llvm python quick-start-build.py

        # Build with custom LLVM location Windows cmd
        > set LLVM_HOME=C:\\path\\to\\llvm && python quick-start-build.py

        # Build with custom LLVM location PowerShell
        > $env:LLVM_HOME = "C:\\path\\to\\llvm"; python quick-start-build.py


    Build Configurations:
        Windows:
            - Build Type: RelWithDebInfo
            - Binary Location: build-{version}/bin/RelWithDebInfo/ispc

        Other Platforms:
            - Build Type: Debug
            - Binary Location: build-{version}/bin/ispc

    Notes:
        - Automatically determines optimal parallel build count
        - Cleans up failed CMake configurations
        - Preserves existing LLVM and build directories if present
        - Uses GitHub API to fetch appropriate LLVM binaries
        - Supports custom LLVM archive URLs via environment variable

    Dependencies:
        - CMake: For building ISPC
        - Python 3.6+: For script execution
        - C++ Compiler: Compatible with chosen LLVM version
        - m4: Required for ISPC build
        - flex, bison: Required for ISPC build
        - onetbb: Required for ISPC build
    """

    # Set up argument parser
    parser = argparse.ArgumentParser(
        description="ISPC build script for downloading and setting up LLVM dependencies, "
                    "configuring the build environment, and running the ISPC build and test suite.",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Environment Variables:
  LLVM_HOME      Directory for LLVM installation (default: current working directory)
  ISPC_HOME      Root directory of ISPC source (default: parent directory of this script)
  ARCHIVE_URL    Direct download URL for LLVM package (only if automatic detection fails)

Examples:
  # Build with default LLVM 20
  $ python quick-start-build.py

  # Build with specific LLVM version
  $ python quick-start-build.py 17

  # Build with custom LLVM location
  $ LLVM_HOME=/path/to/llvm python quick-start-build.py

  # Build with custom LLVM location Windows cmd
  > set LLVM_HOME=C:\\path\\to\\llvm && python quick-start-build.py

  # Build with custom LLVM location PowerShell
  > $env:LLVM_HOME = "C:\\path\\to\\llvm"; python quick-start-build.py
        """
    )

    parser.add_argument("llvm_version", type=str, nargs="?", default="20",
                        help="LLVM version to use (default: 20)")
    args = parser.parse_args()

    # Set up default values and paths
    llvm_version = args.llvm_version
    llvm_home = os.getenv("LLVM_HOME", os.getcwd())
    # Determine number of processors (defaulting to 8 if unknown)
    try:
        nproc = cpu_count() or 8
    except Exception:
        nproc = 8

    scripts_dir = Path(__file__).parent.absolute()
    ispc_root = scripts_dir.parent.absolute()
    ispc_home = Path(os.getenv("ISPC_HOME", str(ispc_root)))
    build_dir = ispc_home / f"build-{llvm_version}"
    llvm_dir = Path(llvm_home) / f"llvm-{llvm_version}"

    arch = platform.machine()
    os_name = platform.system()
    is_windows = os_name == 'Windows'

    print(f"LLVM_HOME: {llvm_home}")
    print(f"ISPC_HOME: {ispc_home}")

    os.chdir(llvm_home)

    if not llvm_dir.exists():
        asset_name, asset_url, version = get_llvm_asset(llvm_version, os_name, arch)

        if not asset_name:
            archive_url = os.getenv("ARCHIVE_URL")
            if archive_url:
                asset_name = os.path.basename(archive_url)
                asset_url = archive_url
                version = re.search(f"{llvm_version}\\.[0-9]*", asset_name).group(0)
            else:
                print("Error: Failed to deduct and fetch LLVM archives from Github API.")
                print("Please set ARCHIVE_URL environment variable to the direct download URL of the LLVM package.")
                print("Example: export ARCHIVE_URL='https://github.com/ispc/ispc.dependencies/releases/download/...'")
                sys.exit(1)

        print(f"Asset Name: {asset_name}")
        print(f"Download URL: {asset_url}")

        if Path(asset_name).exists():
            Path(asset_name).unlink()

        download_file(asset_url, asset_name)
        extract_archive(asset_name, is_windows)

        Path(f"bin-{version}").rename(llvm_dir)
    else:
        print(f"{llvm_dir} already exists")

    build_type = "RelWithDebInfo" if is_windows else "Debug"
    if not build_dir.exists():
        env = os.environ.copy()
        env["PATH"] = f"{llvm_dir / 'bin'}{os.pathsep}{env['PATH']}"

        configure_cmd = [
            "cmake",
            "-B", str(build_dir),
            str(ispc_root),
            f"-DCMAKE_BUILD_TYPE={build_type}",
            "-DISPC_SLIM_BINARY=ON"
        ]
        print("Configure build of ISPC")
        run_command(configure_cmd,
                    lambda: (
                        print(f"CMake failed, cleaning up build directory {build_dir}"),
                        shutil.rmtree(build_dir)
                        ),
                    env=env)
    else:
        print(f"{build_dir} already exists")

    print("Build ISPC")
    build_cmd = ["cmake", "--build", str(build_dir), "--parallel", str(nproc)]
    if is_windows:
        build_cmd.extend(["--config", build_type])
    run_command(build_cmd)

    print("Run ispc --support-matrix")
    ispc_bin = build_dir / "bin"
    ispc_exe = ispc_bin / build_type / "ispc" if is_windows else ispc_bin / "ispc"
    run_command([str(ispc_exe), "--support-matrix"])

    print("Run check-all")
    check_all_cmd = ["cmake", "--build", str(build_dir), "--target", "check-all"]
    if is_windows:
        check_all_cmd.extend(["--config", build_type])
    run_command(check_all_cmd)

if __name__ == "__main__":
    main()