#!/usr/bin/python3 from __future__ import annotations import dataclasses import json import locale import os import os.path import pathlib import subprocess import re import tempfile from typing import cast from debian_linux.config_v2 import ( Config, ConfigMerged, ConfigMergedDebianarch, ConfigMergedFeatureset, ConfigMergedFlavour, ) from debian_linux.dataclasses_deb822 import read_deb822, write_deb822 from debian_linux.debian import \ PackageBuildprofile, \ PackageRelationEntry, PackageRelationGroup, \ VersionLinux, BinaryPackage from debian_linux.gencontrol import Gencontrol as Base, PackagesBundle, \ MakeFlags from debian_linux.utils import Templates locale.setlocale(locale.LC_CTYPE, "C.UTF-8") class Gencontrol(Base): disable_installer: bool disable_signed: bool env_flags = [ ('DEBIAN_KERNEL_DISABLE_INSTALLER', 'disable_installer', 'installer modules'), ('DEBIAN_KERNEL_DISABLE_SIGNED', 'disable_signed', 'signed code'), ] def __init__( self, config_dirs=[ pathlib.Path('debian/config'), pathlib.Path('debian/config.local'), ], template_dirs=["debian/templates"], ) -> None: super().__init__( Config.read_orig(config_dirs).merged, Templates(template_dirs), VersionLinux) self.config_dirs = config_dirs for debianrelease in self.config.debianreleases: if debianrelease.name_regex.fullmatch(self.changelog[0].distribution): self.debianrelease = debianrelease break else: raise RuntimeError( f'No debianrelease config matches {self.changelog[0].distribution}') self.process_changelog() for env, attr, desc in self.env_flags: setattr(self, attr, False) if os.getenv(env): if self.changelog[0].distribution == 'UNRELEASED': import warnings warnings.warn(f'Disable {desc} on request ({env} set)') setattr(self, attr, True) else: raise RuntimeError( f'Unable to disable {desc} in release build ({env} set)') def _setup_makeflags(self, names, makeflags, data) -> None: for src, dst, optional in names: if src in data or not optional: makeflags[dst] = data[src] def do_main_setup( self, config: ConfigMerged, vars: dict[str, str], makeflags: MakeFlags, ) -> None: super().do_main_setup(config, vars, makeflags) makeflags.update({ 'VERSION': self.version.linux_version, 'UPSTREAMVERSION': self.version.linux_upstream, 'ABINAME': self.abiname, 'SOURCEVERSION': self.version.complete, }) makeflags['SOURCE_BASENAME'] = vars['source_basename'] makeflags['SOURCE_SUFFIX'] = vars['source_suffix'] # Prepare to generate debian/tests/control self.tests_control = list(self.templates.get_tests_control('main.tests-control', vars)) def do_main_makefile( self, config: ConfigMerged, vars: dict[str, str], makeflags: MakeFlags, ) -> None: for featureset in self.config.root_featuresets: makeflags_featureset = makeflags.copy() makeflags_featureset['FEATURESET'] = featureset.name self.bundle.makefile.add_rules(f'source_{featureset.name}', 'source', makeflags_featureset) self.bundle.makefile.add_deps('source', [f'source_{featureset.name}']) makeflags = makeflags.copy() makeflags['ALL_FEATURESETS'] = ' '.join(i.name for i in self.config.root_featuresets) super().do_main_makefile(config, vars, makeflags) def do_main_packages( self, config: ConfigMerged, vars: dict[str, str], makeflags: MakeFlags, ) -> None: self.bundle.add('main', (), makeflags, vars) # Only build the metapackages if their names won't exactly match # the packages they depend on do_meta = config.packages.meta \ and vars['source_suffix'] != '-' + vars['version'] if config.packages.docs: self.bundle.add('docs', (), makeflags, vars) if do_meta: self.bundle.add('docs.meta', (), makeflags, vars) if config.packages.source: self.bundle.add('sourcebin', (), makeflags, vars) if do_meta: self.bundle.add('sourcebin.meta', (), makeflags, vars) if config.packages.libc_dev: libcdev_kernelarches = set() libcdev_multiarches = set() for kernelarch in self.config.kernelarchs: libcdev_kernelarches.add(kernelarch.name) for debianarch in kernelarch.debianarchs: libcdev_multiarches.add( f'{debianarch.defs_debianarch.multiarch}:{kernelarch.name}' ) libcdev_makeflags = makeflags.copy() libcdev_makeflags['ALL_LIBCDEV_KERNELARCHES'] = ' '.join(sorted(libcdev_kernelarches)) libcdev_makeflags['ALL_LIBCDEV_MULTIARCHES'] = ' '.join(sorted(libcdev_multiarches)) self.bundle.add('libc-dev', (), libcdev_makeflags, vars) def do_indep_featureset_setup( self, config: ConfigMergedFeatureset, vars: dict[str, str], makeflags: MakeFlags, ) -> None: makeflags['LOCALVERSION'] = vars['localversion'] kernel_arches = set() for kernelarch in self.config.kernelarchs: for debianarch in kernelarch.debianarchs: for featureset in debianarch.featuresets: if config.name_featureset in featureset.name: kernel_arches.add(kernelarch.name) makeflags['ALL_KERNEL_ARCHES'] = ' '.join(sorted(list(kernel_arches))) vars['featureset_desc'] = '' if config.name_featureset != 'none': desc = config.description vars['featureset_desc'] = (' with the %s featureset' % desc.short[desc.parts[0]]) def do_indep_featureset_packages( self, config: ConfigMergedFeatureset, vars: dict[str, str], makeflags: MakeFlags, ) -> None: self.bundle.add('headers.featureset', (config.name_featureset, ), makeflags, vars) def do_arch_setup( self, config: ConfigMergedDebianarch, vars: dict[str, str], makeflags: MakeFlags, ) -> None: makeflags['KERNEL_ARCH'] = config.name_kernelarch def do_arch_packages( self, config: ConfigMergedDebianarch, vars: dict[str, str], makeflags: MakeFlags, ) -> None: arch = config.name if not self.disable_signed: build_signed = config.build.enable_signed else: build_signed = False if build_signed: # Make sure variables remain vars['signedtemplate_binaryversion'] = '@signedtemplate_binaryversion@' vars['signedtemplate_sourceversion'] = '@signedtemplate_sourceversion@' self.bundle.add('signed-template', (arch,), makeflags, vars, arch=arch) bundle_signed = self.bundles[f'signed-{arch}'] = \ PackagesBundle(f'signed-{arch}', 'signed.source.control', vars, self.templates) with bundle_signed.open('source/lintian-overrides', 'w') as f: f.write(self.substitute( self.templates.get('signed.source.lintian-overrides'), vars)) with bundle_signed.open('changelog.head', 'w') as f: dist = self.changelog[0].distribution urgency = self.changelog[0].urgency f.write(f'''\ linux-signed-{vars['arch']} (@signedtemplate_sourceversion@) {dist}; urgency={urgency} * Sign kernel from {self.changelog[0].source} @signedtemplate_binaryversion@ ''') if config.packages.source and list(config.featuresets): self.bundle.add('config', (arch, ), makeflags, vars) if config.packages.tools_unversioned: self.bundle.add('tools-unversioned', (arch, ), makeflags, vars) if config.packages.tools_versioned: self.bundle.add('tools-versioned', (arch, ), makeflags, vars) def do_featureset_setup( self, featureset: ConfigMergedFeatureset, vars: dict[str, str], makeflags: MakeFlags, ) -> None: vars['localversion_headers'] = vars['localversion'] makeflags['LOCALVERSION_HEADERS'] = vars['localversion_headers'] def do_flavour_setup( self, config: ConfigMergedFlavour, vars: dict[str, str], makeflags: MakeFlags, ) -> None: vars['flavour'] = vars['localversion'][1:] vars['class'] = config.description.hardware or '' vars['longclass'] = config.description.hardware_long or vars['class'] vars['localversion-image'] = vars['localversion'] vars['image-stem'] = cast(str, config.build.kernel_stem) if t := config.build.cflags: makeflags['KCFLAGS'] = t makeflags['COMPILER'] = config.build.compiler if t := config.build.compiler_gnutype: makeflags['KERNEL_GNU_TYPE'] = t if t := config.build.compiler_gnutype_compat: makeflags['COMPAT_GNU_TYPE'] = t makeflags['IMAGE_FILE'] = config.build.kernel_file makeflags['IMAGE_INSTALL_STEM'] = config.build.kernel_stem makeflags['LOCALVERSION'] = vars['localversion'] makeflags['LOCALVERSION_IMAGE'] = vars['localversion-image'] def do_flavour_packages( self, config: ConfigMergedFlavour, vars: dict[str, str], makeflags: MakeFlags, ) -> None: arch = config.name_debianarch ruleid = (arch, config.name_featureset, config.name_flavour) packages_headers = ( self.bundle.add('headers', ruleid, makeflags, vars, arch=arch) ) assert len(packages_headers) == 1 do_meta = config.packages.meta relation_compiler = PackageRelationEntry(cast(str, config.build.compiler)) relation_compiler_header = PackageRelationGroup([relation_compiler]) # Generate compiler build-depends for native: # gcc-N [arm64] self.bundle.source.build_depends_arch.merge([ PackageRelationEntry( relation_compiler, arches={arch}, restrictions='', ) ]) # Generate compiler build-depends for cross: # gcc-N-aarch64-linux-gnu [arm64] self.bundle.source.build_depends_arch.merge([ PackageRelationEntry( relation_compiler, name=f'{relation_compiler.name}-{config.defs_debianarch.gnutype_package}', arches={arch}, restrictions='', ) ]) # Generate compiler build-depends for kernel: # gcc-N-hppa64-linux-gnu [hppa] if gnutype := config.build.compiler_gnutype: if gnutype != config.defs_debianarch.gnutype: self.bundle.source.build_depends_arch.merge([ PackageRelationEntry( relation_compiler, name=f'{relation_compiler.name}-{gnutype.replace("_", "-")}', arches={arch}, restrictions='', ) ]) # Generate compiler build-depends for compat: # gcc-arm-linux-gnueabihf [arm64] # XXX: Linux uses various definitions for this, all ending with "gcc", not $CC if gnutype := config.build.compiler_gnutype_compat: if gnutype != config.defs_debianarch.gnutype: self.bundle.source.build_depends_arch.merge([ PackageRelationEntry( f'gcc-{gnutype.replace("_", "-")}', arches={arch}, restrictions='', ) ]) packages_own = [] if not self.disable_signed: build_signed = config.build.enable_signed else: build_signed = False if build_signed: bundle_signed = self.bundles[f'signed-{arch}'] else: bundle_signed = self.bundle vars.setdefault('desc', '') if build_signed: packages_image_unsigned = ( self.bundle.add('image-unsigned', ruleid, makeflags, vars, arch=arch) ) packages_image = packages_image_unsigned[:] packages_image.extend( bundle_signed.add('signed.image', ruleid, makeflags, vars, arch=arch) ) else: packages_image = packages_image_unsigned = ( bundle_signed.add('image', ruleid, makeflags, vars, arch=arch) ) for field in ('Depends', 'Provides', 'Suggests', 'Recommends', 'Conflicts', 'Breaks'): for i in getattr(config.relations.image, field.lower(), []): for package_image in packages_image: getattr(package_image, field.lower()).merge( PackageRelationGroup(i, arches={arch}) ) for field in ('Depends', 'Suggests', 'Recommends'): for i in getattr(config.relations.image, field.lower(), []): group = PackageRelationGroup(i, arches={arch}) for entry in group: if entry.operator is not None: entry.operator = -entry.operator for package_image in packages_image: package_image.breaks.append(PackageRelationGroup([entry])) if desc_parts := config.description.parts: # XXX: Workaround, we need to support multiple entries of the same # name parts = list(set(desc_parts)) parts.sort() for package_image in packages_image: desc = package_image.description for part in parts: desc.append(config.description.long[part]) desc.append_short(config.description.short[part]) packages_headers[0].depends.merge(relation_compiler_header) packages_own.extend(packages_image) packages_own.extend(packages_headers) # The image meta-packages will depend on signed linux-image # packages where applicable, so should be built from the # signed source packages The header meta-packages will also be # built along with the signed packages, to create a dependency # relationship that ensures src:linux and src:linux-signed-* # transition to testing together. if do_meta: packages_meta = ( bundle_signed.add('image.meta', ruleid, makeflags, vars, arch=arch) ) assert len(packages_meta) == 1 packages_meta += ( bundle_signed.add(build_signed and 'signed.headers.meta' or 'headers.meta', ruleid, makeflags, vars, arch=arch) ) assert len(packages_meta) == 2 if ( config.defs_flavour.is_default and not self.vars['source_suffix'] ): packages_meta[0].provides.append('linux-image-generic') packages_meta[1].provides.append('linux-headers-generic') packages_own.extend(packages_meta) if config.build.enable_vdso: makeflags['VDSO'] = True packages_own.extend( self.bundle.add('image-dbg', ruleid, makeflags, vars, arch=arch) ) if do_meta: packages_own.extend( self.bundle.add('image-dbg.meta', ruleid, makeflags, vars, arch=arch) ) if ( config.defs_flavour.is_default # XXX and not self.vars['source_suffix'] ): packages_own.extend( self.bundle.add('image-extra-dev', ruleid, makeflags, vars, arch=arch) ) # In a quick build, only build the quick flavour (if any). if not config.defs_flavour.is_quick: for package in packages_own: package.build_profiles[0].neg.add('pkg.linux.quick') tests_control_image = list( self.templates.get_tests_control('image.tests-control', vars)) for c in tests_control_image: c.depends.extend( [i.name for i in packages_image_unsigned] ) tests_control_headers = list( self.templates.get_tests_control('headers.tests-control', vars)) for c in tests_control_headers: c.depends.extend( [i.name for i in packages_headers] + [i.name for i in packages_image_unsigned] ) self.tests_control.extend(tests_control_image) self.tests_control.extend(tests_control_headers) kconfig = [] for c in config.config: for d in self.config_dirs: if (f := d / c).exists(): kconfig.append(str(f)) makeflags['KCONFIG'] = ' '.join(kconfig) makeflags['KCONFIG_OPTIONS'] = '' # Add "salt" to fix #872263 makeflags['KCONFIG_OPTIONS'] += \ ' -o "BUILD_SALT=\\"%(abiname)s%(localversion)s\\""' % vars merged_config = ('debian/build/config.%s_%s_%s' % (config.name_debianarch, config.name_featureset, config.name_flavour)) self.bundle.makefile.add_cmds(merged_config, ["$(MAKE) -f debian/rules.real %s %s" % (merged_config, makeflags)]) if ( config.name_featureset == 'none' and not self.disable_installer and config.packages.installer ): with tempfile.TemporaryDirectory(prefix='linux-gencontrol') as config_dir: base_path = pathlib.Path('debian/installer').absolute() config_path = pathlib.Path(config_dir) (config_path / 'modules').symlink_to(base_path / 'modules') (config_path / 'package-list').symlink_to(base_path / 'package-list') with (config_path / 'kernel-versions').open('w') as versions: versions.write(f'{arch} - {vars["flavour"]} - - -\n') # Add udebs using kernel-wedge kw_env = os.environ.copy() kw_env['KW_DEFCONFIG_DIR'] = config_dir kw_env['KW_CONFIG_DIR'] = config_dir kw_proc = subprocess.Popen( ['kernel-wedge', 'gen-control', vars['abiname']], stdout=subprocess.PIPE, text=True, env=kw_env) assert kw_proc.stdout is not None udeb_packages_base = list(read_deb822(BinaryPackage, kw_proc.stdout)) kw_proc.wait() if kw_proc.returncode != 0: raise RuntimeError('kernel-wedge exited with code %d' % kw_proc.returncode) udeb_packages = [ dataclasses.replace( package_base, # kernel-wedge currently chokes on Build-Profiles so add it now build_profiles=PackageBuildprofile.parse( '', ), meta_rules_target='installer', ) for package_base in udeb_packages_base ] makeflags_local = makeflags.copy() makeflags_local['IMAGE_PACKAGE_NAME'] = udeb_packages[0].name bundle_signed.add_packages( udeb_packages, (config.name_debianarch, config.name_featureset, config.name_flavour), makeflags_local, arch=arch, ) if build_signed: # XXX This is a hack to exclude the udebs from # the package list while still being able to # convince debhelper and kernel-wedge to go # part way to building them. udeb_packages = [ dataclasses.replace( package_base, # kernel-wedge currently chokes on Build-Profiles so add it now build_profiles=PackageBuildprofile.parse( '', ), meta_rules_target='installer-test', ) for package_base in udeb_packages_base ] self.bundle.add_packages( udeb_packages, (config.name_debianarch, config.name_featureset, config.name_flavour), makeflags_local, arch=arch, check_packages=False, ) def process_changelog(self) -> None: version = self.version = self.changelog[0].version if self.debianrelease.abi_version_full: self.abiname = version.linux_upstream_full \ + self.debianrelease.abi_suffix # All Debian versions must have a distinct ABI version. # So if this is not the first Debian version with its # upstream version and Debian release, distinguish it by # adding a serial number suffix. n = sum(1 for entry in self.changelog if (entry.version.linux_upstream_full == version.linux_upstream_full and self.debianrelease.name_regex.fullmatch(entry.distribution))) if n > 1: self.abiname += f'+{n-1}' else: self.abiname = version.linux_upstream \ + self.debianrelease.abi_suffix self.vars = { 'upstreamversion': self.version.linux_upstream, 'version': self.version.linux_version, 'version_complete': self.version.complete, 'source_basename': re.sub(r'-[\d.]+$', '', self.changelog[0].source), 'source_upstream': self.version.upstream, 'source_package': self.changelog[0].source, 'abiname': self.abiname, } self.vars['source_suffix'] = \ self.changelog[0].source[len(self.vars['source_basename']):] if not self.debianrelease.revision_regex.fullmatch(version.revision): raise RuntimeError( f"Can't upload to {self.changelog[0].distribution} with a version of {version}") def write(self) -> None: super().write() self.write_tests_control() self.write_signed() def write_signed(self) -> None: for bundle in self.bundles.values(): pkg_sign_entries = {} for p in bundle.packages.values(): if not isinstance(p, BinaryPackage): continue if pkg_sign_pkg := p.meta_sign_package: pkg_sign_entries[pkg_sign_pkg] = { 'trusted_certs': [], 'files': [ { 'sig_type': e.split(':', 1)[-1], 'file': e.split(':', 1)[0], } for e in p.meta_sign_files ], } if pkg_sign_entries: with bundle.path('files.json').open('w') as f: json.dump({'packages': pkg_sign_entries}, f, indent=2) def write_tests_control(self) -> None: with open("debian/tests/control", 'w') as f: write_deb822(self.tests_control, f) if __name__ == '__main__': Gencontrol()()