#!/usr/bin/env python # Copyright (C) 2014 Igalia S.L. # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2 of the License, or (at your option) any later version. # # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA from __future__ import print_function from contextlib import closing import argparse import errno import multiprocessing import os import re import shutil import subprocess import tarfile def enum(**enums): return type('Enum', (), enums) class Rule(object): Result = enum(INCLUDE=1, EXCLUDE=2, NO_MATCH=3) def __init__(self, type, pattern): self.type = type self.original_pattern = pattern self.pattern = re.compile(pattern) def test(self, file): if not(self.pattern.search(file)): return Rule.Result.NO_MATCH return self.type class Ruleset(object): _global_rules = None def __init__(self): # By default, accept all files. self.rules = [Rule(Rule.Result.INCLUDE, '.*')] @classmethod def global_rules(cls): if not cls._global_rules: cls._global_rules = Ruleset() return cls._global_rules @classmethod def add_global_rule(cls, rule): cls.global_rules().add_rule(rule) def add_rule(self, rule): self.rules.append(rule) def passes(self, file): allowed = False for rule in self.rules: result = rule.test(file) if result == Rule.Result.NO_MATCH: continue allowed = Rule.Result.INCLUDE == result return allowed class File(object): def __init__(self, source_root, tarball_root): self.source_root = source_root self.tarball_root = tarball_root def should_skip_file(self, path): # Do not skip files explicitly added from the manifest. return False def get_files(self): yield (self.source_root, self.tarball_root) class Directory(object): def __init__(self, source_root, tarball_root): self.source_root = source_root self.tarball_root = tarball_root self.rules = Ruleset() self.files_in_version_control = self.list_files_in_version_control() def add_rule(self, rule): self.rules.add_rule(rule) def get_tarball_path(self, filename): return filename.replace(self.source_root, self.tarball_root, 1) def list_files_in_version_control(self): # FIXME: Only git is supported for now. p = subprocess.Popen(['git', 'ls-tree', '-r', '--name-only', 'HEAD', self.source_root], stdout=subprocess.PIPE) out = p.communicate()[0] if not out: return [] return out.rstrip('\n').split('\n') def should_skip_file(self, path): return path not in self.files_in_version_control def get_files(self): for root, dirs, files in os.walk(self.source_root): def passes_all_rules(entry): return Ruleset.global_rules().passes(entry) and self.rules.passes(entry) to_keep = filter(passes_all_rules, dirs) del dirs[:] dirs.extend(to_keep) for file in files: file = os.path.join(root, file) if not passes_all_rules(file): continue yield (file, self.get_tarball_path(file)) class Manifest(object): def __init__(self, manifest_filename, source_root, build_root, tarball_root='/'): self.current_directory = None self.directories = [] self.tarball_root = tarball_root self.source_root = source_root self.build_root = build_root # Normalize the tarball root so that it starts and ends with a slash. if not self.tarball_root.endswith('/'): self.tarball_root = self.tarball_root + '/' if not self.tarball_root.startswith('/'): self.tarball_root = '/' + self.tarball_root with open(manifest_filename, 'r') as file: for line in file.readlines(): self.process_line(line) def add_rule(self, rule): if self.current_directory is not None: self.current_directory.add_rule(rule) else: Ruleset.add_global_rule(rule) def add_directory(self, directory): self.current_directory = directory self.directories.append(directory) def resolve_variables(self, string, strip=False): if strip: return string.replace('$source', '').replace('$build', '') string = string.replace('$source', self.source_root) if self.build_root: string = string.replace('$build', self.build_root) elif string.find('$build') != -1: raise Exception('Manifest has $build but build root not given.') return string def get_full_source_path(self, source_path): full_source_path = self.resolve_variables(source_path) if not os.path.exists(full_source_path): full_source_path = os.path.join(self.source_root, source_path) if not os.path.exists(full_source_path): raise Exception('Could not find directory %s' % full_source_path) return full_source_path def get_full_tarball_path(self, path): path = self.resolve_variables(path, strip=True) return self.tarball_root + path def get_source_and_tarball_paths_from_parts(self, parts): full_source_path = self.get_full_source_path(parts[1]) if len(parts) > 2: full_tarball_path = self.get_full_tarball_path(parts[2]) else: full_tarball_path = self.get_full_tarball_path(parts[1]) return (full_source_path, full_tarball_path) def process_line(self, line): parts = line.split() if not parts: return if parts[0].startswith("#"): return if parts[0] == "directory" and len(parts) > 1: self.add_directory(Directory(*self.get_source_and_tarball_paths_from_parts(parts))) elif parts[0] == "file" and len(parts) > 1: self.add_directory(File(*self.get_source_and_tarball_paths_from_parts(parts))) elif parts[0] == "exclude" and len(parts) > 1: self.add_rule(Rule(Rule.Result.EXCLUDE, self.resolve_variables(parts[1]))) elif parts[0] == "include" and len(parts) > 1: self.add_rule(Rule(Rule.Result.INCLUDE, self.resolve_variables(parts[1]))) def should_skip_file(self, directory, filename): # Only allow files that are not in version control when they are explicitly included in the manifest from the build dir. if filename.startswith(self.build_root): return False return directory.should_skip_file(filename) def get_files(self): for directory in self.directories: for file_tuple in directory.get_files(): if self.should_skip_file(directory, file_tuple[0]): continue yield file_tuple def create_tarfile(self, output): count = 0 for file_tuple in self.get_files(): count = count + 1 with closing(tarfile.open(output, 'w')) as tarball: for i, (file_path, tarball_path) in enumerate(self.get_files(), start=1): print('Tarring file {0} of {1}'.format(i, count).ljust(40), end='\r') tarball.add(file_path, tarball_path) print("Wrote {0}".format(output).ljust(40)) class Distcheck(object): BUILD_DIRECTORY_NAME = "_build" INSTALL_DIRECTORY_NAME = "_install" def __init__(self, source_root, build_root): self.source_root = source_root self.build_root = build_root def extract_tarball(self, tarball_path): with closing(tarfile.open(tarball_path, 'r')) as tarball: tarball.extractall(self.build_root) def configure(self, dist_dir, build_dir, install_dir): def create_dir(directory, directory_type): try: os.mkdir(directory) except OSError, e: if e.errno != errno.EEXIST or not os.path.isdir(directory): raise Exception("Could not create %s dir at %s: %s" % (directory_type, directory, str(e))) create_dir(build_dir, "build") create_dir(install_dir, "install") command = ['cmake', '-DPORT=GTK', '-DCMAKE_INSTALL_PREFIX=%s' % install_dir, '-DCMAKE_BUILD_TYPE=Release', dist_dir] subprocess.check_call(command, cwd=build_dir) def build(self, build_dir): command = ['make'] make_args = os.getenv('MAKE_ARGS') if make_args: command.extend(make_args.split(' ')) else: command.append('-j%d' % multiprocessing.cpu_count()) subprocess.check_call(command, cwd=build_dir) def install(self, build_dir): subprocess.check_call(['make', 'install'], cwd=build_dir) def clean(self, dist_dir): shutil.rmtree(dist_dir) def check(self, tarball): tarball_name, ext = os.path.splitext(os.path.basename(tarball)) dist_dir = os.path.join(self.build_root, tarball_name) build_dir = os.path.join(dist_dir, self.BUILD_DIRECTORY_NAME) install_dir = os.path.join(dist_dir, self.INSTALL_DIRECTORY_NAME) self.extract_tarball(tarball) self.configure(dist_dir, build_dir, install_dir) self.build(build_dir) self.install(build_dir) self.clean(dist_dir) if __name__ == "__main__": class FilePathAction(argparse.Action): def __call__(self, parser, namespace, values, option_string=None): setattr(namespace, self.dest, os.path.abspath(values)) def ensure_version_if_possible(arguments): if arguments.version is not None: return pkgconfig_file = os.path.join(arguments.build_dir, "Source/WebKit2/webkit2gtk-4.0.pc") if os.path.isfile(pkgconfig_file): p = subprocess.Popen(['pkg-config', '--modversion', pkgconfig_file], stdout=subprocess.PIPE) version = p.communicate()[0] if version: arguments.version = version.rstrip('\n') def get_tarball_root_and_output_filename_from_arguments(arguments): tarball_root = "webkitgtk" if arguments.version is not None: tarball_root += '-' + arguments.version output_filename = os.path.join(arguments.build_dir, tarball_root + ".tar") return tarball_root, output_filename parser = argparse.ArgumentParser(description='Build a distribution bundle.') parser.add_argument('-c', '--check', action='store_true', help='Check the tarball') parser.add_argument('-s', '--source-dir', type=str, action=FilePathAction, default=os.getcwd(), help='The top-level directory of the source distribution. ' + \ 'Directory for relative paths. Defaults to current directory.') parser.add_argument('--version', type=str, default=None, help='The version of the tarball to generate') parser.add_argument('-b', '--build-dir', type=str, action=FilePathAction, default=os.getcwd(), help='The top-level path of directory of the build root. ' + \ 'By default is the current directory.') parser.add_argument('manifest_filename', metavar="manifest", type=str, action=FilePathAction, help='The path to the manifest file.') arguments = parser.parse_args() # Paths in the manifest are relative to the source directory, and this script assumes that # current working directory is the source directory, so change the current working directory # to be the source directory. os.chdir(arguments.source_dir) ensure_version_if_possible(arguments) tarball_root, output_filename = get_tarball_root_and_output_filename_from_arguments(arguments) manifest = Manifest(arguments.manifest_filename, arguments.source_dir, arguments.build_dir, tarball_root) manifest.create_tarfile(output_filename) if arguments.check: Distcheck(arguments.source_dir, arguments.build_dir).check(output_filename)