## @file # Create makefile for MS nmake and GNU make # # Copyright (c) 2019, Intel Corporation. All rights reserved.
# SPDX-License-Identifier: BSD-2-Clause-Patent # from __future__ import absolute_import from AutoGen.AutoGen import AutoGen from Common.LongFilePathSupport import LongFilePath, CopyLongFilePath from Common.BuildToolError import * from Common.DataType import * from Common.Misc import * from Common.StringUtils import NormPath,GetSplitList from collections import defaultdict from Workspace.WorkspaceCommon import OrderedListDict import os.path as path import copy import hashlib from . import InfSectionParser from . import GenC from . import GenMake from . import GenDepex from io import BytesIO from GenPatchPcdTable.GenPatchPcdTable import parsePcdInfoFromMapFile from Workspace.MetaFileCommentParser import UsageList from .GenPcdDb import CreatePcdDatabaseCode from Common.caching import cached_class_function from AutoGen.ModuleAutoGenHelper import PlatformInfo,WorkSpaceInfo import json import tempfile ## Mapping Makefile type gMakeTypeMap = {TAB_COMPILER_MSFT:"nmake", "GCC":"gmake"} # # Regular expression for finding Include Directories, the difference between MSFT and INTEL/GCC # is the former use /I , the Latter used -I to specify include directories # gBuildOptIncludePatternMsft = re.compile(r"(?:.*?)/I[ \t]*([^ ]*)", re.MULTILINE | re.DOTALL) gBuildOptIncludePatternOther = re.compile(r"(?:.*?)-I[ \t]*([^ ]*)", re.MULTILINE | re.DOTALL) ## default file name for AutoGen gAutoGenCodeFileName = "AutoGen.c" gAutoGenHeaderFileName = "AutoGen.h" gAutoGenStringFileName = "%(module_name)sStrDefs.h" gAutoGenStringFormFileName = "%(module_name)sStrDefs.hpk" gAutoGenDepexFileName = "%(module_name)s.depex" gAutoGenImageDefFileName = "%(module_name)sImgDefs.h" gAutoGenIdfFileName = "%(module_name)sIdf.hpk" gInfSpecVersion = "0x00010017" # # Match name = variable # gEfiVarStoreNamePattern = re.compile("\s*name\s*=\s*(\w+)") # # The format of guid in efivarstore statement likes following and must be correct: # guid = {0xA04A27f4, 0xDF00, 0x4D42, {0xB5, 0x52, 0x39, 0x51, 0x13, 0x02, 0x11, 0x3D}} # gEfiVarStoreGuidPattern = re.compile("\s*guid\s*=\s*({.*?{.*?}\s*})") # # Template string to generic AsBuilt INF # gAsBuiltInfHeaderString = TemplateString("""${header_comments} # DO NOT EDIT # FILE auto-generated [Defines] INF_VERSION = ${module_inf_version} BASE_NAME = ${module_name} FILE_GUID = ${module_guid} MODULE_TYPE = ${module_module_type}${BEGIN} VERSION_STRING = ${module_version_string}${END}${BEGIN} PCD_IS_DRIVER = ${pcd_is_driver_string}${END}${BEGIN} UEFI_SPECIFICATION_VERSION = ${module_uefi_specification_version}${END}${BEGIN} PI_SPECIFICATION_VERSION = ${module_pi_specification_version}${END}${BEGIN} ENTRY_POINT = ${module_entry_point}${END}${BEGIN} UNLOAD_IMAGE = ${module_unload_image}${END}${BEGIN} CONSTRUCTOR = ${module_constructor}${END}${BEGIN} DESTRUCTOR = ${module_destructor}${END}${BEGIN} SHADOW = ${module_shadow}${END}${BEGIN} PCI_VENDOR_ID = ${module_pci_vendor_id}${END}${BEGIN} PCI_DEVICE_ID = ${module_pci_device_id}${END}${BEGIN} PCI_CLASS_CODE = ${module_pci_class_code}${END}${BEGIN} PCI_REVISION = ${module_pci_revision}${END}${BEGIN} BUILD_NUMBER = ${module_build_number}${END}${BEGIN} SPEC = ${module_spec}${END}${BEGIN} UEFI_HII_RESOURCE_SECTION = ${module_uefi_hii_resource_section}${END}${BEGIN} MODULE_UNI_FILE = ${module_uni_file}${END} [Packages.${module_arch}]${BEGIN} ${package_item}${END} [Binaries.${module_arch}]${BEGIN} ${binary_item}${END} [PatchPcd.${module_arch}]${BEGIN} ${patchablepcd_item} ${END} [Protocols.${module_arch}]${BEGIN} ${protocol_item} ${END} [Ppis.${module_arch}]${BEGIN} ${ppi_item} ${END} [Guids.${module_arch}]${BEGIN} ${guid_item} ${END} [PcdEx.${module_arch}]${BEGIN} ${pcd_item} ${END} [LibraryClasses.${module_arch}] ## @LIB_INSTANCES${BEGIN} # ${libraryclasses_item}${END} ${depexsection_item} ${userextension_tianocore_item} ${tail_comments} [BuildOptions.${module_arch}] ## @AsBuilt${BEGIN} ## ${flags_item}${END} """) # # extend lists contained in a dictionary with lists stored in another dictionary # if CopyToDict is not derived from DefaultDict(list) then this may raise exception # def ExtendCopyDictionaryLists(CopyToDict, CopyFromDict): for Key in CopyFromDict: CopyToDict[Key].extend(CopyFromDict[Key]) # Create a directory specified by a set of path elements and return the full path def _MakeDir(PathList): RetVal = path.join(*PathList) CreateDirectory(RetVal) return RetVal # # Convert string to C format array # def _ConvertStringToByteArray(Value): Value = Value.strip() if not Value: return None if Value[0] == '{': if not Value.endswith('}'): return None Value = Value.replace(' ', '').replace('{', '').replace('}', '') ValFields = Value.split(',') try: for Index in range(len(ValFields)): ValFields[Index] = str(int(ValFields[Index], 0)) except ValueError: return None Value = '{' + ','.join(ValFields) + '}' return Value Unicode = False if Value.startswith('L"'): if not Value.endswith('"'): return None Value = Value[1:] Unicode = True elif not Value.startswith('"') or not Value.endswith('"'): return None Value = eval(Value) # translate escape character NewValue = '{' for Index in range(0, len(Value)): if Unicode: NewValue = NewValue + str(ord(Value[Index]) % 0x10000) + ',' else: NewValue = NewValue + str(ord(Value[Index]) % 0x100) + ',' Value = NewValue + '0}' return Value ## ModuleAutoGen class # # This class encapsules the AutoGen behaviors for the build tools. In addition to # the generation of AutoGen.h and AutoGen.c, it will generate *.depex file according # to the [depex] section in module's inf file. # class ModuleAutoGen(AutoGen): # call super().__init__ then call the worker function with different parameter count def __init__(self, Workspace, MetaFile, Target, Toolchain, Arch, *args, **kwargs): if not hasattr(self, "_Init"): self._InitWorker(Workspace, MetaFile, Target, Toolchain, Arch, *args) self._Init = True ## Cache the timestamps of metafiles of every module in a class attribute # TimeDict = {} def __new__(cls, Workspace, MetaFile, Target, Toolchain, Arch, *args, **kwargs): # check if this module is employed by active platform if not PlatformInfo(Workspace, args[0], Target, Toolchain, Arch,args[-1]).ValidModule(MetaFile): EdkLogger.verbose("Module [%s] for [%s] is not employed by active platform\n" \ % (MetaFile, Arch)) return None return super(ModuleAutoGen, cls).__new__(cls, Workspace, MetaFile, Target, Toolchain, Arch, *args, **kwargs) ## Initialize ModuleAutoGen # # @param Workspace EdkIIWorkspaceBuild object # @param ModuleFile The path of module file # @param Target Build target (DEBUG, RELEASE) # @param Toolchain Name of tool chain # @param Arch The arch the module supports # @param PlatformFile Platform meta-file # def _InitWorker(self, Workspace, ModuleFile, Target, Toolchain, Arch, PlatformFile,DataPipe): EdkLogger.debug(EdkLogger.DEBUG_9, "AutoGen module [%s] [%s]" % (ModuleFile, Arch)) GlobalData.gProcessingFile = "%s [%s, %s, %s]" % (ModuleFile, Arch, Toolchain, Target) self.Workspace = Workspace self.WorkspaceDir = "" self.PlatformInfo = None self.DataPipe = DataPipe self.__init_platform_info__() self.MetaFile = ModuleFile self.SourceDir = self.MetaFile.SubDir self.SourceDir = mws.relpath(self.SourceDir, self.WorkspaceDir) self.ToolChain = Toolchain self.BuildTarget = Target self.Arch = Arch self.ToolChainFamily = self.PlatformInfo.ToolChainFamily self.BuildRuleFamily = self.PlatformInfo.BuildRuleFamily self.IsCodeFileCreated = False self.IsAsBuiltInfCreated = False self.DepexGenerated = False self.BuildDatabase = self.Workspace.BuildDatabase self.BuildRuleOrder = None self.BuildTime = 0 self._GuidComments = OrderedListDict() self._ProtocolComments = OrderedListDict() self._PpiComments = OrderedListDict() self._BuildTargets = None self._IntroBuildTargetList = None self._FinalBuildTargetList = None self._FileTypes = None self.AutoGenDepSet = set() self.ReferenceModules = [] self.ConstPcd = {} self.FileDependCache = {} def __init_platform_info__(self): pinfo = self.DataPipe.Get("P_Info") self.WorkspaceDir = pinfo.get("WorkspaceDir") self.PlatformInfo = PlatformInfo(self.Workspace,pinfo.get("ActivePlatform"),pinfo.get("Target"),pinfo.get("ToolChain"),pinfo.get("Arch"),self.DataPipe) ## hash() operator of ModuleAutoGen # # The module file path and arch string will be used to represent # hash value of this object # # @retval int Hash value of the module file path and arch # @cached_class_function def __hash__(self): return hash((self.MetaFile, self.Arch, self.ToolChain,self.BuildTarget)) def __repr__(self): return "%s [%s]" % (self.MetaFile, self.Arch) # Get FixedAtBuild Pcds of this Module @cached_property def FixedAtBuildPcds(self): RetVal = [] for Pcd in self.ModulePcdList: if Pcd.Type != TAB_PCDS_FIXED_AT_BUILD: continue if Pcd not in RetVal: RetVal.append(Pcd) return RetVal @cached_property def FixedVoidTypePcds(self): RetVal = {} for Pcd in self.FixedAtBuildPcds: if Pcd.DatumType == TAB_VOID: if '.'.join((Pcd.TokenSpaceGuidCName, Pcd.TokenCName)) not in RetVal: RetVal['.'.join((Pcd.TokenSpaceGuidCName, Pcd.TokenCName))] = Pcd.DefaultValue return RetVal @property def UniqueBaseName(self): ModuleNames = self.DataPipe.Get("M_Name") if not ModuleNames: return self.Name return ModuleNames.get((self.Name,self.MetaFile),self.Name) # Macros could be used in build_rule.txt (also Makefile) @cached_property def Macros(self): return OrderedDict(( ("WORKSPACE" ,self.WorkspaceDir), ("MODULE_NAME" ,self.Name), ("MODULE_NAME_GUID" ,self.UniqueBaseName), ("MODULE_GUID" ,self.Guid), ("MODULE_VERSION" ,self.Version), ("MODULE_TYPE" ,self.ModuleType), ("MODULE_FILE" ,str(self.MetaFile)), ("MODULE_FILE_BASE_NAME" ,self.MetaFile.BaseName), ("MODULE_RELATIVE_DIR" ,self.SourceDir), ("MODULE_DIR" ,self.SourceDir), ("BASE_NAME" ,self.Name), ("ARCH" ,self.Arch), ("TOOLCHAIN" ,self.ToolChain), ("TOOLCHAIN_TAG" ,self.ToolChain), ("TOOL_CHAIN_TAG" ,self.ToolChain), ("TARGET" ,self.BuildTarget), ("BUILD_DIR" ,self.PlatformInfo.BuildDir), ("BIN_DIR" ,os.path.join(self.PlatformInfo.BuildDir, self.Arch)), ("LIB_DIR" ,os.path.join(self.PlatformInfo.BuildDir, self.Arch)), ("MODULE_BUILD_DIR" ,self.BuildDir), ("OUTPUT_DIR" ,self.OutputDir), ("DEBUG_DIR" ,self.DebugDir), ("DEST_DIR_OUTPUT" ,self.OutputDir), ("DEST_DIR_DEBUG" ,self.DebugDir), ("PLATFORM_NAME" ,self.PlatformInfo.Name), ("PLATFORM_GUID" ,self.PlatformInfo.Guid), ("PLATFORM_VERSION" ,self.PlatformInfo.Version), ("PLATFORM_RELATIVE_DIR" ,self.PlatformInfo.SourceDir), ("PLATFORM_DIR" ,mws.join(self.WorkspaceDir, self.PlatformInfo.SourceDir)), ("PLATFORM_OUTPUT_DIR" ,self.PlatformInfo.OutputDir), ("FFS_OUTPUT_DIR" ,self.FfsOutputDir) )) ## Return the module build data object @cached_property def Module(self): return self.BuildDatabase[self.MetaFile, self.Arch, self.BuildTarget, self.ToolChain] ## Return the module name @cached_property def Name(self): return self.Module.BaseName ## Return the module DxsFile if exist @cached_property def DxsFile(self): return self.Module.DxsFile ## Return the module meta-file GUID @cached_property def Guid(self): # # To build same module more than once, the module path with FILE_GUID overridden has # the file name FILE_GUIDmodule.inf, but the relative path (self.MetaFile.File) is the real path # in DSC. The overridden GUID can be retrieved from file name # if os.path.basename(self.MetaFile.File) != os.path.basename(self.MetaFile.Path): # # Length of GUID is 36 # return os.path.basename(self.MetaFile.Path)[:36] return self.Module.Guid ## Return the module version @cached_property def Version(self): return self.Module.Version ## Return the module type @cached_property def ModuleType(self): return self.Module.ModuleType ## Return the component type (for Edk.x style of module) @cached_property def ComponentType(self): return self.Module.ComponentType ## Return the build type @cached_property def BuildType(self): return self.Module.BuildType ## Return the PCD_IS_DRIVER setting @cached_property def PcdIsDriver(self): return self.Module.PcdIsDriver ## Return the autogen version, i.e. module meta-file version @cached_property def AutoGenVersion(self): return self.Module.AutoGenVersion ## Check if the module is library or not @cached_property def IsLibrary(self): return bool(self.Module.LibraryClass) ## Check if the module is binary module or not @cached_property def IsBinaryModule(self): return self.Module.IsBinaryModule ## Return the directory to store intermediate files of the module @cached_property def BuildDir(self): return _MakeDir(( self.PlatformInfo.BuildDir, self.Arch, self.SourceDir, self.MetaFile.BaseName )) ## Return the directory to store the intermediate object files of the module @cached_property def OutputDir(self): return _MakeDir((self.BuildDir, "OUTPUT")) ## Return the directory path to store ffs file @cached_property def FfsOutputDir(self): if GlobalData.gFdfParser: return path.join(self.PlatformInfo.BuildDir, TAB_FV_DIRECTORY, "Ffs", self.Guid + self.Name) return '' ## Return the directory to store auto-gened source files of the module @cached_property def DebugDir(self): return _MakeDir((self.BuildDir, "DEBUG")) ## Return the path of custom file @cached_property def CustomMakefile(self): RetVal = {} for Type in self.Module.CustomMakefile: MakeType = gMakeTypeMap[Type] if Type in gMakeTypeMap else 'nmake' File = os.path.join(self.SourceDir, self.Module.CustomMakefile[Type]) RetVal[MakeType] = File return RetVal ## Return the directory of the makefile # # @retval string The directory string of module's makefile # @cached_property def MakeFileDir(self): return self.BuildDir ## Return build command string # # @retval string Build command string # @cached_property def BuildCommand(self): return self.PlatformInfo.BuildCommand ## Get Module package and Platform package # # @retval list The list of package object # @cached_property def PackageList(self): PkagList = [] if self.Module.Packages: PkagList.extend(self.Module.Packages) Platform = self.BuildDatabase[self.PlatformInfo.MetaFile, self.Arch, self.BuildTarget, self.ToolChain] for Package in Platform.Packages: if Package in PkagList: continue PkagList.append(Package) return PkagList ## Get object list of all packages the module and its dependent libraries belong to and the Platform depends on # # @retval list The list of package object # @cached_property def DerivedPackageList(self): PackageList = [] PackageList.extend(self.PackageList) for M in self.DependentLibraryList: for Package in M.Packages: if Package in PackageList: continue PackageList.append(Package) return PackageList ## Get the depex string # # @return : a string contain all depex expression. def _GetDepexExpresionString(self): DepexStr = '' DepexList = [] ## DPX_SOURCE IN Define section. if self.Module.DxsFile: return DepexStr for M in [self.Module] + self.DependentLibraryList: Filename = M.MetaFile.Path InfObj = InfSectionParser.InfSectionParser(Filename) DepexExpressionList = InfObj.GetDepexExpresionList() for DepexExpression in DepexExpressionList: for key in DepexExpression: Arch, ModuleType = key DepexExpr = [x for x in DepexExpression[key] if not str(x).startswith('#')] # the type of build module is USER_DEFINED. # All different DEPEX section tags would be copied into the As Built INF file # and there would be separate DEPEX section tags if self.ModuleType.upper() == SUP_MODULE_USER_DEFINED or self.ModuleType.upper() == SUP_MODULE_HOST_APPLICATION: if (Arch.upper() == self.Arch.upper()) and (ModuleType.upper() != TAB_ARCH_COMMON): DepexList.append({(Arch, ModuleType): DepexExpr}) else: if Arch.upper() == TAB_ARCH_COMMON or \ (Arch.upper() == self.Arch.upper() and \ ModuleType.upper() in [TAB_ARCH_COMMON, self.ModuleType.upper()]): DepexList.append({(Arch, ModuleType): DepexExpr}) #the type of build module is USER_DEFINED. if self.ModuleType.upper() == SUP_MODULE_USER_DEFINED or self.ModuleType.upper() == SUP_MODULE_HOST_APPLICATION: for Depex in DepexList: for key in Depex: DepexStr += '[Depex.%s.%s]\n' % key DepexStr += '\n'.join('# '+ val for val in Depex[key]) DepexStr += '\n\n' if not DepexStr: return '[Depex.%s]\n' % self.Arch return DepexStr #the type of build module not is USER_DEFINED. Count = 0 for Depex in DepexList: Count += 1 if DepexStr != '': DepexStr += ' AND ' DepexStr += '(' for D in Depex.values(): DepexStr += ' '.join(val for val in D) Index = DepexStr.find('END') if Index > -1 and Index == len(DepexStr) - 3: DepexStr = DepexStr[:-3] DepexStr = DepexStr.strip() DepexStr += ')' if Count == 1: DepexStr = DepexStr.lstrip('(').rstrip(')').strip() if not DepexStr: return '[Depex.%s]\n' % self.Arch return '[Depex.%s]\n# ' % self.Arch + DepexStr ## Merge dependency expression # # @retval list The token list of the dependency expression after parsed # @cached_property def DepexList(self): if self.DxsFile or self.IsLibrary or TAB_DEPENDENCY_EXPRESSION_FILE in self.FileTypes: return {} DepexList = [] # # Append depex from dependent libraries, if not "BEFORE", "AFTER" expression # FixedVoidTypePcds = {} for M in [self] + self.LibraryAutoGenList: FixedVoidTypePcds.update(M.FixedVoidTypePcds) for M in [self] + self.LibraryAutoGenList: Inherited = False for D in M.Module.Depex[self.Arch, self.ModuleType]: if DepexList != []: DepexList.append('AND') DepexList.append('(') #replace D with value if D is FixedAtBuild PCD NewList = [] for item in D: if '.' not in item: NewList.append(item) else: try: Value = FixedVoidTypePcds[item] if len(Value.split(',')) != 16: EdkLogger.error("build", FORMAT_INVALID, "{} used in [Depex] section should be used as FixedAtBuild type and VOID* datum type and 16 bytes in the module.".format(item)) NewList.append(Value) except: EdkLogger.error("build", FORMAT_INVALID, "{} used in [Depex] section should be used as FixedAtBuild type and VOID* datum type in the module.".format(item)) DepexList.extend(NewList) if DepexList[-1] == 'END': # no need of a END at this time DepexList.pop() DepexList.append(')') Inherited = True if Inherited: EdkLogger.verbose("DEPEX[%s] (+%s) = %s" % (self.Name, M.Module.BaseName, DepexList)) if 'BEFORE' in DepexList or 'AFTER' in DepexList: break if len(DepexList) > 0: EdkLogger.verbose('') return {self.ModuleType:DepexList} ## Merge dependency expression # # @retval list The token list of the dependency expression after parsed # @cached_property def DepexExpressionDict(self): if self.DxsFile or self.IsLibrary or TAB_DEPENDENCY_EXPRESSION_FILE in self.FileTypes: return {} DepexExpressionString = '' # # Append depex from dependent libraries, if not "BEFORE", "AFTER" expresion # for M in [self.Module] + self.DependentLibraryList: Inherited = False for D in M.DepexExpression[self.Arch, self.ModuleType]: if DepexExpressionString != '': DepexExpressionString += ' AND ' DepexExpressionString += '(' DepexExpressionString += D DepexExpressionString = DepexExpressionString.rstrip('END').strip() DepexExpressionString += ')' Inherited = True if Inherited: EdkLogger.verbose("DEPEX[%s] (+%s) = %s" % (self.Name, M.BaseName, DepexExpressionString)) if 'BEFORE' in DepexExpressionString or 'AFTER' in DepexExpressionString: break if len(DepexExpressionString) > 0: EdkLogger.verbose('') return {self.ModuleType:DepexExpressionString} # Get the tiano core user extension, it is contain dependent library. # @retval: a list contain tiano core userextension. # def _GetTianoCoreUserExtensionList(self): TianoCoreUserExtentionList = [] for M in [self.Module] + self.DependentLibraryList: Filename = M.MetaFile.Path InfObj = InfSectionParser.InfSectionParser(Filename) TianoCoreUserExtenList = InfObj.GetUserExtensionTianoCore() for TianoCoreUserExtent in TianoCoreUserExtenList: for Section in TianoCoreUserExtent: ItemList = Section.split(TAB_SPLIT) Arch = self.Arch if len(ItemList) == 4: Arch = ItemList[3] if Arch.upper() == TAB_ARCH_COMMON or Arch.upper() == self.Arch.upper(): TianoCoreList = [] TianoCoreList.extend([TAB_SECTION_START + Section + TAB_SECTION_END]) TianoCoreList.extend(TianoCoreUserExtent[Section][:]) TianoCoreList.append('\n') TianoCoreUserExtentionList.append(TianoCoreList) return TianoCoreUserExtentionList ## Return the list of specification version required for the module # # @retval list The list of specification defined in module file # @cached_property def Specification(self): return self.Module.Specification ## Tool option for the module build # # @param PlatformInfo The object of PlatformBuildInfo # @retval dict The dict containing valid options # @cached_property def BuildOption(self): RetVal, self.BuildRuleOrder = self.PlatformInfo.ApplyBuildOption(self.Module) if self.BuildRuleOrder: self.BuildRuleOrder = ['.%s' % Ext for Ext in self.BuildRuleOrder.split()] return RetVal ## Get include path list from tool option for the module build # # @retval list The include path list # @cached_property def BuildOptionIncPathList(self): # # Regular expression for finding Include Directories, the difference between MSFT and INTEL/GCC # is the former use /I , the Latter used -I to specify include directories # if self.PlatformInfo.ToolChainFamily in (TAB_COMPILER_MSFT): BuildOptIncludeRegEx = gBuildOptIncludePatternMsft elif self.PlatformInfo.ToolChainFamily in ('INTEL', 'GCC'): BuildOptIncludeRegEx = gBuildOptIncludePatternOther else: # # New ToolChainFamily, don't known whether there is option to specify include directories # return [] RetVal = [] for Tool in ('CC', 'PP', 'VFRPP', 'ASLPP', 'ASLCC', 'APP', 'ASM'): try: FlagOption = self.BuildOption[Tool]['FLAGS'] except KeyError: FlagOption = '' IncPathList = [NormPath(Path, self.Macros) for Path in BuildOptIncludeRegEx.findall(FlagOption)] # # EDK II modules must not reference header files outside of the packages they depend on or # within the module's directory tree. Report error if violation. # if GlobalData.gDisableIncludePathCheck == False: for Path in IncPathList: if (Path not in self.IncludePathList) and (CommonPath([Path, self.MetaFile.Dir]) != self.MetaFile.Dir): ErrMsg = "The include directory for the EDK II module in this line is invalid %s specified in %s FLAGS '%s'" % (Path, Tool, FlagOption) EdkLogger.error("build", PARAMETER_INVALID, ExtraData=ErrMsg, File=str(self.MetaFile)) RetVal += IncPathList return RetVal ## Return a list of files which can be built from source # # What kind of files can be built is determined by build rules in # $(CONF_DIRECTORY)/build_rule.txt and toolchain family. # @cached_property def SourceFileList(self): RetVal = [] ToolChainTagSet = {"", TAB_STAR, self.ToolChain} ToolChainFamilySet = {"", TAB_STAR, self.ToolChainFamily, self.BuildRuleFamily} for F in self.Module.Sources: # match tool chain if F.TagName not in ToolChainTagSet: EdkLogger.debug(EdkLogger.DEBUG_9, "The toolchain [%s] for processing file [%s] is found, " "but [%s] is currently used" % (F.TagName, str(F), self.ToolChain)) continue # match tool chain family or build rule family if F.ToolChainFamily not in ToolChainFamilySet: EdkLogger.debug( EdkLogger.DEBUG_0, "The file [%s] must be built by tools of [%s], " \ "but current toolchain family is [%s], buildrule family is [%s]" \ % (str(F), F.ToolChainFamily, self.ToolChainFamily, self.BuildRuleFamily)) continue # add the file path into search path list for file including if F.Dir not in self.IncludePathList: self.IncludePathList.insert(0, F.Dir) RetVal.append(F) self._MatchBuildRuleOrder(RetVal) for F in RetVal: self._ApplyBuildRule(F, TAB_UNKNOWN_FILE) return RetVal def _MatchBuildRuleOrder(self, FileList): Order_Dict = {} self.BuildOption for SingleFile in FileList: if self.BuildRuleOrder and SingleFile.Ext in self.BuildRuleOrder and SingleFile.Ext in self.BuildRules: key = SingleFile.Path.rsplit(SingleFile.Ext,1)[0] if key in Order_Dict: Order_Dict[key].append(SingleFile.Ext) else: Order_Dict[key] = [SingleFile.Ext] RemoveList = [] for F in Order_Dict: if len(Order_Dict[F]) > 1: Order_Dict[F].sort(key=lambda i: self.BuildRuleOrder.index(i)) for Ext in Order_Dict[F][1:]: RemoveList.append(F + Ext) for item in RemoveList: FileList.remove(item) return FileList ## Return the list of unicode files @cached_property def UnicodeFileList(self): return self.FileTypes.get(TAB_UNICODE_FILE,[]) ## Return the list of vfr files @cached_property def VfrFileList(self): return self.FileTypes.get(TAB_VFR_FILE, []) ## Return the list of Image Definition files @cached_property def IdfFileList(self): return self.FileTypes.get(TAB_IMAGE_FILE,[]) ## Return a list of files which can be built from binary # # "Build" binary files are just to copy them to build directory. # # @retval list The list of files which can be built later # @cached_property def BinaryFileList(self): RetVal = [] for F in self.Module.Binaries: if F.Target not in [TAB_ARCH_COMMON, TAB_STAR] and F.Target != self.BuildTarget: continue RetVal.append(F) self._ApplyBuildRule(F, F.Type, BinaryFileList=RetVal) return RetVal @cached_property def BuildRules(self): RetVal = {} BuildRuleDatabase = self.PlatformInfo.BuildRule for Type in BuildRuleDatabase.FileTypeList: #first try getting build rule by BuildRuleFamily RuleObject = BuildRuleDatabase[Type, self.BuildType, self.Arch, self.BuildRuleFamily] if not RuleObject: # build type is always module type, but ... if self.ModuleType != self.BuildType: RuleObject = BuildRuleDatabase[Type, self.ModuleType, self.Arch, self.BuildRuleFamily] #second try getting build rule by ToolChainFamily if not RuleObject: RuleObject = BuildRuleDatabase[Type, self.BuildType, self.Arch, self.ToolChainFamily] if not RuleObject: # build type is always module type, but ... if self.ModuleType != self.BuildType: RuleObject = BuildRuleDatabase[Type, self.ModuleType, self.Arch, self.ToolChainFamily] if not RuleObject: continue RuleObject = RuleObject.Instantiate(self.Macros) RetVal[Type] = RuleObject for Ext in RuleObject.SourceFileExtList: RetVal[Ext] = RuleObject return RetVal def _ApplyBuildRule(self, File, FileType, BinaryFileList=None): if self._BuildTargets is None: self._IntroBuildTargetList = set() self._FinalBuildTargetList = set() self._BuildTargets = defaultdict(set) self._FileTypes = defaultdict(set) if not BinaryFileList: BinaryFileList = self.BinaryFileList SubDirectory = os.path.join(self.OutputDir, File.SubDir) if not os.path.exists(SubDirectory): CreateDirectory(SubDirectory) TargetList = set() FinalTargetName = set() RuleChain = set() SourceList = [File] Index = 0 # # Make sure to get build rule order value # self.BuildOption while Index < len(SourceList): # Reset the FileType if not the first iteration. if Index > 0: FileType = TAB_UNKNOWN_FILE Source = SourceList[Index] Index = Index + 1 if Source != File: CreateDirectory(Source.Dir) if File.IsBinary and File == Source and File in BinaryFileList: # Skip all files that are not binary libraries if not self.IsLibrary: continue RuleObject = self.BuildRules[TAB_DEFAULT_BINARY_FILE] elif FileType in self.BuildRules: RuleObject = self.BuildRules[FileType] elif Source.Ext in self.BuildRules: RuleObject = self.BuildRules[Source.Ext] else: # No more rule to apply: Source is a final target. FinalTargetName.add(Source) continue FileType = RuleObject.SourceFileType self._FileTypes[FileType].add(Source) # stop at STATIC_LIBRARY for library if self.IsLibrary and FileType == TAB_STATIC_LIBRARY: FinalTargetName.add(Source) continue Target = RuleObject.Apply(Source, self.BuildRuleOrder) if not Target: # No Target: Source is a final target. FinalTargetName.add(Source) continue TargetList.add(Target) self._BuildTargets[FileType].add(Target) if not Source.IsBinary and Source == File: self._IntroBuildTargetList.add(Target) # to avoid cyclic rule if FileType in RuleChain: EdkLogger.error("build", ERROR_STATEMENT, "Cyclic dependency detected while generating rule for %s" % str(Source)) RuleChain.add(FileType) SourceList.extend(Target.Outputs) # For each final target name, retrieve the corresponding TargetDescBlock instance. for FTargetName in FinalTargetName: for Target in TargetList: if FTargetName == Target.Target: self._FinalBuildTargetList.add(Target) @cached_property def Targets(self): if self._BuildTargets is None: self._IntroBuildTargetList = set() self._FinalBuildTargetList = set() self._BuildTargets = defaultdict(set) self._FileTypes = defaultdict(set) #TRICK: call SourceFileList property to apply build rule for source files self.SourceFileList #TRICK: call _GetBinaryFileList to apply build rule for binary files self.BinaryFileList return self._BuildTargets @cached_property def IntroTargetList(self): self.Targets return self._IntroBuildTargetList @cached_property def CodaTargetList(self): self.Targets return self._FinalBuildTargetList @cached_property def FileTypes(self): self.Targets return self._FileTypes ## Get the list of package object the module depends on and the Platform depends on # # @retval list The package object list # @cached_property def DependentPackageList(self): return self.PackageList ## Return the list of auto-generated code file # # @retval list The list of auto-generated file # @cached_property def AutoGenFileList(self): AutoGenUniIdf = self.BuildType != 'UEFI_HII' UniStringBinBuffer = BytesIO() IdfGenBinBuffer = BytesIO() RetVal = {} AutoGenC = TemplateString() AutoGenH = TemplateString() StringH = TemplateString() StringIdf = TemplateString() GenC.CreateCode(self, AutoGenC, AutoGenH, StringH, AutoGenUniIdf, UniStringBinBuffer, StringIdf, AutoGenUniIdf, IdfGenBinBuffer) # # AutoGen.c is generated if there are library classes in inf, or there are object files # if str(AutoGenC) != "" and (len(self.Module.LibraryClasses) > 0 or TAB_OBJECT_FILE in self.FileTypes): AutoFile = PathClass(gAutoGenCodeFileName, self.DebugDir) RetVal[AutoFile] = str(AutoGenC) self._ApplyBuildRule(AutoFile, TAB_UNKNOWN_FILE) if str(AutoGenH) != "": AutoFile = PathClass(gAutoGenHeaderFileName, self.DebugDir) RetVal[AutoFile] = str(AutoGenH) self._ApplyBuildRule(AutoFile, TAB_UNKNOWN_FILE) if str(StringH) != "": AutoFile = PathClass(gAutoGenStringFileName % {"module_name":self.Name}, self.DebugDir) RetVal[AutoFile] = str(StringH) self._ApplyBuildRule(AutoFile, TAB_UNKNOWN_FILE) if UniStringBinBuffer is not None and UniStringBinBuffer.getvalue() != b"": AutoFile = PathClass(gAutoGenStringFormFileName % {"module_name":self.Name}, self.OutputDir) RetVal[AutoFile] = UniStringBinBuffer.getvalue() AutoFile.IsBinary = True self._ApplyBuildRule(AutoFile, TAB_UNKNOWN_FILE) if UniStringBinBuffer is not None: UniStringBinBuffer.close() if str(StringIdf) != "": AutoFile = PathClass(gAutoGenImageDefFileName % {"module_name":self.Name}, self.DebugDir) RetVal[AutoFile] = str(StringIdf) self._ApplyBuildRule(AutoFile, TAB_UNKNOWN_FILE) if IdfGenBinBuffer is not None and IdfGenBinBuffer.getvalue() != b"": AutoFile = PathClass(gAutoGenIdfFileName % {"module_name":self.Name}, self.OutputDir) RetVal[AutoFile] = IdfGenBinBuffer.getvalue() AutoFile.IsBinary = True self._ApplyBuildRule(AutoFile, TAB_UNKNOWN_FILE) if IdfGenBinBuffer is not None: IdfGenBinBuffer.close() return RetVal ## Return the list of library modules explicitly or implicitly used by this module @cached_property def DependentLibraryList(self): # only merge library classes and PCD for non-library module if self.IsLibrary: return [] return self.PlatformInfo.ApplyLibraryInstance(self.Module) ## Get the list of PCDs from current module # # @retval list The list of PCD # @cached_property def ModulePcdList(self): # apply PCD settings from platform RetVal = self.PlatformInfo.ApplyPcdSetting(self, self.Module.Pcds) return RetVal @cached_property def _PcdComments(self): ReVal = OrderedListDict() ExtendCopyDictionaryLists(ReVal, self.Module.PcdComments) if not self.IsLibrary: for Library in self.DependentLibraryList: ExtendCopyDictionaryLists(ReVal, Library.PcdComments) return ReVal ## Get the list of PCDs from dependent libraries # # @retval list The list of PCD # @cached_property def LibraryPcdList(self): if self.IsLibrary: return [] RetVal = [] Pcds = set() # get PCDs from dependent libraries for Library in self.DependentLibraryList: PcdsInLibrary = OrderedDict() for Key in Library.Pcds: # skip duplicated PCDs if Key in self.Module.Pcds or Key in Pcds: continue Pcds.add(Key) PcdsInLibrary[Key] = copy.copy(Library.Pcds[Key]) RetVal.extend(self.PlatformInfo.ApplyPcdSetting(self, PcdsInLibrary, Library=Library)) return RetVal ## Get the GUID value mapping # # @retval dict The mapping between GUID cname and its value # @cached_property def GuidList(self): RetVal = self.Module.Guids for Library in self.DependentLibraryList: RetVal.update(Library.Guids) ExtendCopyDictionaryLists(self._GuidComments, Library.GuidComments) ExtendCopyDictionaryLists(self._GuidComments, self.Module.GuidComments) return RetVal @cached_property def GetGuidsUsedByPcd(self): RetVal = OrderedDict(self.Module.GetGuidsUsedByPcd()) for Library in self.DependentLibraryList: RetVal.update(Library.GetGuidsUsedByPcd()) return RetVal ## Get the protocol value mapping # # @retval dict The mapping between protocol cname and its value # @cached_property def ProtocolList(self): RetVal = OrderedDict(self.Module.Protocols) for Library in self.DependentLibraryList: RetVal.update(Library.Protocols) ExtendCopyDictionaryLists(self._ProtocolComments, Library.ProtocolComments) ExtendCopyDictionaryLists(self._ProtocolComments, self.Module.ProtocolComments) return RetVal ## Get the PPI value mapping # # @retval dict The mapping between PPI cname and its value # @cached_property def PpiList(self): RetVal = OrderedDict(self.Module.Ppis) for Library in self.DependentLibraryList: RetVal.update(Library.Ppis) ExtendCopyDictionaryLists(self._PpiComments, Library.PpiComments) ExtendCopyDictionaryLists(self._PpiComments, self.Module.PpiComments) return RetVal ## Get the list of include search path # # @retval list The list path # @cached_property def IncludePathList(self): RetVal = [] RetVal.append(self.MetaFile.Dir) RetVal.append(self.DebugDir) for Package in self.PackageList: PackageDir = mws.join(self.WorkspaceDir, Package.MetaFile.Dir) if PackageDir not in RetVal: RetVal.append(PackageDir) IncludesList = Package.Includes if Package._PrivateIncludes: if not self.MetaFile.OriginalPath.Path.startswith(PackageDir): IncludesList = list(set(Package.Includes).difference(set(Package._PrivateIncludes))) for Inc in IncludesList: if Inc not in RetVal: RetVal.append(str(Inc)) RetVal.extend(self.IncPathFromBuildOptions) return RetVal @cached_property def IncPathFromBuildOptions(self): IncPathList = [] for tool in self.BuildOption: if 'FLAGS' in self.BuildOption[tool]: flags = self.BuildOption[tool]['FLAGS'] whitespace = False for flag in flags.split(" "): flag = flag.strip() if flag.startswith(("/I","-I")): if len(flag)>2: if os.path.exists(flag[2:]): IncPathList.append(flag[2:]) else: whitespace = True continue if whitespace and flag: if os.path.exists(flag): IncPathList.append(flag) whitespace = False return IncPathList @cached_property def IncludePathLength(self): return sum(len(inc)+1 for inc in self.IncludePathList) ## Get the list of include paths from the packages # # @IncludesList list The list path # @cached_property def PackageIncludePathList(self): IncludesList = [] for Package in self.PackageList: PackageDir = mws.join(self.WorkspaceDir, Package.MetaFile.Dir) IncludesList = Package.Includes if Package._PrivateIncludes: if not self.MetaFile.Path.startswith(PackageDir): IncludesList = list(set(Package.Includes).difference(set(Package._PrivateIncludes))) return IncludesList ## Get HII EX PCDs which maybe used by VFR # # efivarstore used by VFR may relate with HII EX PCDs # Get the variable name and GUID from efivarstore and HII EX PCD # List the HII EX PCDs in As Built INF if both name and GUID match. # # @retval list HII EX PCDs # def _GetPcdsMaybeUsedByVfr(self): if not self.SourceFileList: return [] NameGuids = set() for SrcFile in self.SourceFileList: if SrcFile.Ext.lower() != '.vfr': continue Vfri = os.path.join(self.OutputDir, SrcFile.BaseName + '.i') if not os.path.exists(Vfri): continue VfriFile = open(Vfri, 'r') Content = VfriFile.read() VfriFile.close() Pos = Content.find('efivarstore') while Pos != -1: # # Make sure 'efivarstore' is the start of efivarstore statement # In case of the value of 'name' (name = efivarstore) is equal to 'efivarstore' # Index = Pos - 1 while Index >= 0 and Content[Index] in ' \t\r\n': Index -= 1 if Index >= 0 and Content[Index] != ';': Pos = Content.find('efivarstore', Pos + len('efivarstore')) continue # # 'efivarstore' must be followed by name and guid # Name = gEfiVarStoreNamePattern.search(Content, Pos) if not Name: break Guid = gEfiVarStoreGuidPattern.search(Content, Pos) if not Guid: break NameArray = _ConvertStringToByteArray('L"' + Name.group(1) + '"') NameGuids.add((NameArray, GuidStructureStringToGuidString(Guid.group(1)))) Pos = Content.find('efivarstore', Name.end()) if not NameGuids: return [] HiiExPcds = [] for Pcd in self.PlatformInfo.Pcds.values(): if Pcd.Type != TAB_PCDS_DYNAMIC_EX_HII: continue for SkuInfo in Pcd.SkuInfoList.values(): Value = GuidValue(SkuInfo.VariableGuid, self.PlatformInfo.PackageList, self.MetaFile.Path) if not Value: continue Name = _ConvertStringToByteArray(SkuInfo.VariableName) Guid = GuidStructureStringToGuidString(Value) if (Name, Guid) in NameGuids and Pcd not in HiiExPcds: HiiExPcds.append(Pcd) break return HiiExPcds def _GenOffsetBin(self): VfrUniBaseName = {} for SourceFile in self.Module.Sources: if SourceFile.Type.upper() == ".VFR" : # # search the .map file to find the offset of vfr binary in the PE32+/TE file. # VfrUniBaseName[SourceFile.BaseName] = (SourceFile.BaseName + "Bin") elif SourceFile.Type.upper() == ".UNI" : # # search the .map file to find the offset of Uni strings binary in the PE32+/TE file. # VfrUniBaseName["UniOffsetName"] = (self.Name + "Strings") if not VfrUniBaseName: return None MapFileName = os.path.join(self.OutputDir, self.Name + ".map") EfiFileName = os.path.join(self.OutputDir, self.Name + ".efi") VfrUniOffsetList = GetVariableOffset(MapFileName, EfiFileName, list(VfrUniBaseName.values())) if not VfrUniOffsetList: return None OutputName = '%sOffset.bin' % self.Name UniVfrOffsetFileName = os.path.join( self.OutputDir, OutputName) try: fInputfile = open(UniVfrOffsetFileName, "wb+", 0) except: EdkLogger.error("build", FILE_OPEN_FAILURE, "File open failed for %s" % UniVfrOffsetFileName, None) # Use a instance of BytesIO to cache data fStringIO = BytesIO() for Item in VfrUniOffsetList: if (Item[0].find("Strings") != -1): # # UNI offset in image. # GUID + Offset # { 0x8913c5e0, 0x33f6, 0x4d86, { 0x9b, 0xf1, 0x43, 0xef, 0x89, 0xfc, 0x6, 0x66 } } # UniGuid = b'\xe0\xc5\x13\x89\xf63\x86M\x9b\xf1C\xef\x89\xfc\x06f' fStringIO.write(UniGuid) UniValue = pack ('Q', int (Item[1], 16)) fStringIO.write (UniValue) else: # # VFR binary offset in image. # GUID + Offset # { 0xd0bc7cb4, 0x6a47, 0x495f, { 0xaa, 0x11, 0x71, 0x7, 0x46, 0xda, 0x6, 0xa2 } }; # VfrGuid = b'\xb4|\xbc\xd0Gj_I\xaa\x11q\x07F\xda\x06\xa2' fStringIO.write(VfrGuid) VfrValue = pack ('Q', int (Item[1], 16)) fStringIO.write (VfrValue) # # write data into file. # try : fInputfile.write (fStringIO.getvalue()) except: EdkLogger.error("build", FILE_WRITE_FAILURE, "Write data to file %s failed, please check whether the " "file been locked or using by other applications." %UniVfrOffsetFileName, None) fStringIO.close () fInputfile.close () return OutputName @cached_property def OutputFile(self): retVal = set() for Root, Dirs, Files in os.walk(self.BuildDir): for File in Files: # lib file is already added through above CodaTargetList, skip it here if not (File.lower().endswith('.obj') or File.lower().endswith('.debug')): NewFile = path.join(Root, File) retVal.add(NewFile) for Root, Dirs, Files in os.walk(self.FfsOutputDir): for File in Files: NewFile = path.join(Root, File) retVal.add(NewFile) return retVal ## Create AsBuilt INF file the module # def CreateAsBuiltInf(self): if self.IsAsBuiltInfCreated: return # Skip INF file generation for libraries if self.IsLibrary: return # Skip the following code for modules with no source files if not self.SourceFileList: return # Skip the following code for modules without any binary files if self.BinaryFileList: return ### TODO: How to handles mixed source and binary modules # Find all DynamicEx and PatchableInModule PCDs used by this module and dependent libraries # Also find all packages that the DynamicEx PCDs depend on Pcds = [] PatchablePcds = [] Packages = [] PcdCheckList = [] PcdTokenSpaceList = [] for Pcd in self.ModulePcdList + self.LibraryPcdList: if Pcd.Type == TAB_PCDS_PATCHABLE_IN_MODULE: PatchablePcds.append(Pcd) PcdCheckList.append((Pcd.TokenCName, Pcd.TokenSpaceGuidCName, TAB_PCDS_PATCHABLE_IN_MODULE)) elif Pcd.Type in PCD_DYNAMIC_EX_TYPE_SET: if Pcd not in Pcds: Pcds.append(Pcd) PcdCheckList.append((Pcd.TokenCName, Pcd.TokenSpaceGuidCName, TAB_PCDS_DYNAMIC_EX)) PcdCheckList.append((Pcd.TokenCName, Pcd.TokenSpaceGuidCName, TAB_PCDS_DYNAMIC)) PcdTokenSpaceList.append(Pcd.TokenSpaceGuidCName) GuidList = OrderedDict(self.GuidList) for TokenSpace in self.GetGuidsUsedByPcd: # If token space is not referred by patch PCD or Ex PCD, remove the GUID from GUID list # The GUIDs in GUIDs section should really be the GUIDs in source INF or referred by Ex an patch PCDs if TokenSpace not in PcdTokenSpaceList and TokenSpace in GuidList: GuidList.pop(TokenSpace) CheckList = (GuidList, self.PpiList, self.ProtocolList, PcdCheckList) for Package in self.DerivedPackageList: if Package in Packages: continue BeChecked = (Package.Guids, Package.Ppis, Package.Protocols, Package.Pcds) Found = False for Index in range(len(BeChecked)): for Item in CheckList[Index]: if Item in BeChecked[Index]: Packages.append(Package) Found = True break if Found: break VfrPcds = self._GetPcdsMaybeUsedByVfr() for Pkg in self.PlatformInfo.PackageList: if Pkg in Packages: continue for VfrPcd in VfrPcds: if ((VfrPcd.TokenCName, VfrPcd.TokenSpaceGuidCName, TAB_PCDS_DYNAMIC_EX) in Pkg.Pcds or (VfrPcd.TokenCName, VfrPcd.TokenSpaceGuidCName, TAB_PCDS_DYNAMIC) in Pkg.Pcds): Packages.append(Pkg) break ModuleType = SUP_MODULE_DXE_DRIVER if self.ModuleType == SUP_MODULE_UEFI_DRIVER and self.DepexGenerated else self.ModuleType DriverType = self.PcdIsDriver if self.PcdIsDriver else '' Guid = self.Guid MDefs = self.Module.Defines AsBuiltInfDict = { 'module_name' : self.Name, 'module_guid' : Guid, 'module_module_type' : ModuleType, 'module_version_string' : [MDefs['VERSION_STRING']] if 'VERSION_STRING' in MDefs else [], 'pcd_is_driver_string' : [], 'module_uefi_specification_version' : [], 'module_pi_specification_version' : [], 'module_entry_point' : self.Module.ModuleEntryPointList, 'module_unload_image' : self.Module.ModuleUnloadImageList, 'module_constructor' : self.Module.ConstructorList, 'module_destructor' : self.Module.DestructorList, 'module_shadow' : [MDefs['SHADOW']] if 'SHADOW' in MDefs else [], 'module_pci_vendor_id' : [MDefs['PCI_VENDOR_ID']] if 'PCI_VENDOR_ID' in MDefs else [], 'module_pci_device_id' : [MDefs['PCI_DEVICE_ID']] if 'PCI_DEVICE_ID' in MDefs else [], 'module_pci_class_code' : [MDefs['PCI_CLASS_CODE']] if 'PCI_CLASS_CODE' in MDefs else [], 'module_pci_revision' : [MDefs['PCI_REVISION']] if 'PCI_REVISION' in MDefs else [], 'module_build_number' : [MDefs['BUILD_NUMBER']] if 'BUILD_NUMBER' in MDefs else [], 'module_spec' : [MDefs['SPEC']] if 'SPEC' in MDefs else [], 'module_uefi_hii_resource_section' : [MDefs['UEFI_HII_RESOURCE_SECTION']] if 'UEFI_HII_RESOURCE_SECTION' in MDefs else [], 'module_uni_file' : [MDefs['MODULE_UNI_FILE']] if 'MODULE_UNI_FILE' in MDefs else [], 'module_arch' : self.Arch, 'package_item' : [Package.MetaFile.File.replace('\\', '/') for Package in Packages], 'binary_item' : [], 'patchablepcd_item' : [], 'pcd_item' : [], 'protocol_item' : [], 'ppi_item' : [], 'guid_item' : [], 'flags_item' : [], 'libraryclasses_item' : [] } if 'MODULE_UNI_FILE' in MDefs: UNIFile = os.path.join(self.MetaFile.Dir, MDefs['MODULE_UNI_FILE']) if os.path.isfile(UNIFile): shutil.copy2(UNIFile, self.OutputDir) if self.AutoGenVersion > int(gInfSpecVersion, 0): AsBuiltInfDict['module_inf_version'] = '0x%08x' % self.AutoGenVersion else: AsBuiltInfDict['module_inf_version'] = gInfSpecVersion if DriverType: AsBuiltInfDict['pcd_is_driver_string'].append(DriverType) if 'UEFI_SPECIFICATION_VERSION' in self.Specification: AsBuiltInfDict['module_uefi_specification_version'].append(self.Specification['UEFI_SPECIFICATION_VERSION']) if 'PI_SPECIFICATION_VERSION' in self.Specification: AsBuiltInfDict['module_pi_specification_version'].append(self.Specification['PI_SPECIFICATION_VERSION']) OutputDir = self.OutputDir.replace('\\', '/').strip('/') DebugDir = self.DebugDir.replace('\\', '/').strip('/') for Item in self.CodaTargetList: File = Item.Target.Path.replace('\\', '/').strip('/').replace(DebugDir, '').replace(OutputDir, '').strip('/') if os.path.isabs(File): File = File.replace('\\', '/').strip('/').replace(OutputDir, '').strip('/') if Item.Target.Ext.lower() == '.aml': AsBuiltInfDict['binary_item'].append('ASL|' + File) elif Item.Target.Ext.lower() == '.acpi': AsBuiltInfDict['binary_item'].append('ACPI|' + File) elif Item.Target.Ext.lower() == '.efi': AsBuiltInfDict['binary_item'].append('PE32|' + self.Name + '.efi') else: AsBuiltInfDict['binary_item'].append('BIN|' + File) if not self.DepexGenerated: DepexFile = os.path.join(self.OutputDir, self.Name + '.depex') if os.path.exists(DepexFile): self.DepexGenerated = True if self.DepexGenerated: if self.ModuleType in [SUP_MODULE_PEIM]: AsBuiltInfDict['binary_item'].append('PEI_DEPEX|' + self.Name + '.depex') elif self.ModuleType in [SUP_MODULE_DXE_DRIVER, SUP_MODULE_DXE_RUNTIME_DRIVER, SUP_MODULE_DXE_SAL_DRIVER, SUP_MODULE_UEFI_DRIVER]: AsBuiltInfDict['binary_item'].append('DXE_DEPEX|' + self.Name + '.depex') elif self.ModuleType in [SUP_MODULE_DXE_SMM_DRIVER]: AsBuiltInfDict['binary_item'].append('SMM_DEPEX|' + self.Name + '.depex') Bin = self._GenOffsetBin() if Bin: AsBuiltInfDict['binary_item'].append('BIN|%s' % Bin) for Root, Dirs, Files in os.walk(OutputDir): for File in Files: if File.lower().endswith('.pdb'): AsBuiltInfDict['binary_item'].append('DISPOSABLE|' + File) HeaderComments = self.Module.HeaderComments StartPos = 0 for Index in range(len(HeaderComments)): if HeaderComments[Index].find('@BinaryHeader') != -1: HeaderComments[Index] = HeaderComments[Index].replace('@BinaryHeader', '@file') StartPos = Index break AsBuiltInfDict['header_comments'] = '\n'.join(HeaderComments[StartPos:]).replace(':#', '://') AsBuiltInfDict['tail_comments'] = '\n'.join(self.Module.TailComments) GenList = [ (self.ProtocolList, self._ProtocolComments, 'protocol_item'), (self.PpiList, self._PpiComments, 'ppi_item'), (GuidList, self._GuidComments, 'guid_item') ] for Item in GenList: for CName in Item[0]: Comments = '\n '.join(Item[1][CName]) if CName in Item[1] else '' Entry = Comments + '\n ' + CName if Comments else CName AsBuiltInfDict[Item[2]].append(Entry) PatchList = parsePcdInfoFromMapFile( os.path.join(self.OutputDir, self.Name + '.map'), os.path.join(self.OutputDir, self.Name + '.efi') ) if PatchList: for Pcd in PatchablePcds: TokenCName = Pcd.TokenCName for PcdItem in GlobalData.MixedPcd: if (Pcd.TokenCName, Pcd.TokenSpaceGuidCName) in GlobalData.MixedPcd[PcdItem]: TokenCName = PcdItem[0] break for PatchPcd in PatchList: if TokenCName == PatchPcd[0]: break else: continue PcdValue = '' if Pcd.DatumType == 'BOOLEAN': BoolValue = Pcd.DefaultValue.upper() if BoolValue == 'TRUE': Pcd.DefaultValue = '1' elif BoolValue == 'FALSE': Pcd.DefaultValue = '0' if Pcd.DatumType in TAB_PCD_NUMERIC_TYPES: HexFormat = '0x%02x' if Pcd.DatumType == TAB_UINT16: HexFormat = '0x%04x' elif Pcd.DatumType == TAB_UINT32: HexFormat = '0x%08x' elif Pcd.DatumType == TAB_UINT64: HexFormat = '0x%016x' PcdValue = HexFormat % int(Pcd.DefaultValue, 0) else: if Pcd.MaxDatumSize is None or Pcd.MaxDatumSize == '': EdkLogger.error("build", AUTOGEN_ERROR, "Unknown [MaxDatumSize] of PCD [%s.%s]" % (Pcd.TokenSpaceGuidCName, TokenCName) ) ArraySize = int(Pcd.MaxDatumSize, 0) PcdValue = Pcd.DefaultValue if PcdValue[0] != '{': Unicode = False if PcdValue[0] == 'L': Unicode = True PcdValue = PcdValue.lstrip('L') PcdValue = eval(PcdValue) NewValue = '{' for Index in range(0, len(PcdValue)): if Unicode: CharVal = ord(PcdValue[Index]) NewValue = NewValue + '0x%02x' % (CharVal & 0x00FF) + ', ' \ + '0x%02x' % (CharVal >> 8) + ', ' else: NewValue = NewValue + '0x%02x' % (ord(PcdValue[Index]) % 0x100) + ', ' Padding = '0x00, ' if Unicode: Padding = Padding * 2 ArraySize = ArraySize // 2 if ArraySize < (len(PcdValue) + 1): if Pcd.MaxSizeUserSet: EdkLogger.error("build", AUTOGEN_ERROR, "The maximum size of VOID* type PCD '%s.%s' is less than its actual size occupied." % (Pcd.TokenSpaceGuidCName, TokenCName) ) else: ArraySize = len(PcdValue) + 1 if ArraySize > len(PcdValue) + 1: NewValue = NewValue + Padding * (ArraySize - len(PcdValue) - 1) PcdValue = NewValue + Padding.strip().rstrip(',') + '}' elif len(PcdValue.split(',')) <= ArraySize: PcdValue = PcdValue.rstrip('}') + ', 0x00' * (ArraySize - len(PcdValue.split(','))) PcdValue += '}' else: if Pcd.MaxSizeUserSet: EdkLogger.error("build", AUTOGEN_ERROR, "The maximum size of VOID* type PCD '%s.%s' is less than its actual size occupied." % (Pcd.TokenSpaceGuidCName, TokenCName) ) else: ArraySize = len(PcdValue) + 1 PcdItem = '%s.%s|%s|0x%X' % \ (Pcd.TokenSpaceGuidCName, TokenCName, PcdValue, PatchPcd[1]) PcdComments = '' if (Pcd.TokenSpaceGuidCName, Pcd.TokenCName) in self._PcdComments: PcdComments = '\n '.join(self._PcdComments[Pcd.TokenSpaceGuidCName, Pcd.TokenCName]) if PcdComments: PcdItem = PcdComments + '\n ' + PcdItem AsBuiltInfDict['patchablepcd_item'].append(PcdItem) for Pcd in Pcds + VfrPcds: PcdCommentList = [] HiiInfo = '' TokenCName = Pcd.TokenCName for PcdItem in GlobalData.MixedPcd: if (Pcd.TokenCName, Pcd.TokenSpaceGuidCName) in GlobalData.MixedPcd[PcdItem]: TokenCName = PcdItem[0] break if Pcd.Type == TAB_PCDS_DYNAMIC_EX_HII: for SkuName in Pcd.SkuInfoList: SkuInfo = Pcd.SkuInfoList[SkuName] HiiInfo = '## %s|%s|%s' % (SkuInfo.VariableName, SkuInfo.VariableGuid, SkuInfo.VariableOffset) break if (Pcd.TokenSpaceGuidCName, Pcd.TokenCName) in self._PcdComments: PcdCommentList = self._PcdComments[Pcd.TokenSpaceGuidCName, Pcd.TokenCName][:] if HiiInfo: UsageIndex = -1 UsageStr = '' for Index, Comment in enumerate(PcdCommentList): for Usage in UsageList: if Comment.find(Usage) != -1: UsageStr = Usage UsageIndex = Index break if UsageIndex != -1: PcdCommentList[UsageIndex] = '## %s %s %s' % (UsageStr, HiiInfo, PcdCommentList[UsageIndex].replace(UsageStr, '')) else: PcdCommentList.append('## UNDEFINED ' + HiiInfo) PcdComments = '\n '.join(PcdCommentList) PcdEntry = Pcd.TokenSpaceGuidCName + '.' + TokenCName if PcdComments: PcdEntry = PcdComments + '\n ' + PcdEntry AsBuiltInfDict['pcd_item'].append(PcdEntry) for Item in self.BuildOption: if 'FLAGS' in self.BuildOption[Item]: AsBuiltInfDict['flags_item'].append('%s:%s_%s_%s_%s_FLAGS = %s' % (self.ToolChainFamily, self.BuildTarget, self.ToolChain, self.Arch, Item, self.BuildOption[Item]['FLAGS'].strip())) # Generated LibraryClasses section in comments. for Library in self.LibraryAutoGenList: AsBuiltInfDict['libraryclasses_item'].append(Library.MetaFile.File.replace('\\', '/')) # Generated UserExtensions TianoCore section. # All tianocore user extensions are copied. UserExtStr = '' for TianoCore in self._GetTianoCoreUserExtensionList(): UserExtStr += '\n'.join(TianoCore) ExtensionFile = os.path.join(self.MetaFile.Dir, TianoCore[1]) if os.path.isfile(ExtensionFile): shutil.copy2(ExtensionFile, self.OutputDir) AsBuiltInfDict['userextension_tianocore_item'] = UserExtStr # Generated depex expression section in comments. DepexExpression = self._GetDepexExpresionString() AsBuiltInfDict['depexsection_item'] = DepexExpression if DepexExpression else '' AsBuiltInf = TemplateString() AsBuiltInf.Append(gAsBuiltInfHeaderString.Replace(AsBuiltInfDict)) SaveFileOnChange(os.path.join(self.OutputDir, self.Name + '.inf'), str(AsBuiltInf), False) self.IsAsBuiltInfCreated = True def CacheCopyFile(self, DestDir, SourceDir, File): if os.path.isdir(File): return sub_dir = os.path.relpath(File, SourceDir) destination_file = os.path.join(DestDir, sub_dir) destination_dir = os.path.dirname(destination_file) CreateDirectory(destination_dir) try: CopyFileOnChange(File, destination_dir) except: EdkLogger.quiet("[cache warning]: fail to copy file:%s to folder:%s" % (File, destination_dir)) return def CopyModuleToCache(self): # Find the MakeHashStr and PreMakeHashStr from latest MakeHashFileList # and PreMakeHashFileList files MakeHashStr = None PreMakeHashStr = None MakeTimeStamp = 0 PreMakeTimeStamp = 0 Files = [f for f in os.listdir(LongFilePath(self.BuildDir)) if path.isfile(LongFilePath(path.join(self.BuildDir, f)))] for File in Files: if ".MakeHashFileList." in File: #find lastest file through time stamp FileTimeStamp = os.stat(LongFilePath(path.join(self.BuildDir, File)))[8] if FileTimeStamp > MakeTimeStamp: MakeTimeStamp = FileTimeStamp MakeHashStr = File.split('.')[-1] if len(MakeHashStr) != 32: EdkLogger.quiet("[cache error]: wrong MakeHashFileList file:%s" % (File)) if ".PreMakeHashFileList." in File: FileTimeStamp = os.stat(LongFilePath(path.join(self.BuildDir, File)))[8] if FileTimeStamp > PreMakeTimeStamp: PreMakeTimeStamp = FileTimeStamp PreMakeHashStr = File.split('.')[-1] if len(PreMakeHashStr) != 32: EdkLogger.quiet("[cache error]: wrong PreMakeHashFileList file:%s" % (File)) if not MakeHashStr: EdkLogger.quiet("[cache error]: No MakeHashFileList file for module:%s[%s]" % (self.MetaFile.Path, self.Arch)) return if not PreMakeHashStr: EdkLogger.quiet("[cache error]: No PreMakeHashFileList file for module:%s[%s]" % (self.MetaFile.Path, self.Arch)) return # Create Cache destination dirs FileDir = path.join(GlobalData.gBinCacheDest, self.PlatformInfo.OutputDir, self.BuildTarget + "_" + self.ToolChain, self.Arch, self.SourceDir, self.MetaFile.BaseName) FfsDir = path.join(GlobalData.gBinCacheDest, self.PlatformInfo.OutputDir, self.BuildTarget + "_" + self.ToolChain, TAB_FV_DIRECTORY, "Ffs", self.Guid + self.Name) CacheFileDir = path.join(FileDir, MakeHashStr) CacheFfsDir = path.join(FfsDir, MakeHashStr) CreateDirectory (CacheFileDir) CreateDirectory (CacheFfsDir) # Create ModuleHashPair file to support multiple version cache together ModuleHashPair = path.join(FileDir, self.Name + ".ModuleHashPair") ModuleHashPairList = [] # tuple list: [tuple(PreMakefileHash, MakeHash)] if os.path.exists(ModuleHashPair): with open(ModuleHashPair, 'r') as f: ModuleHashPairList = json.load(f) if not (PreMakeHashStr, MakeHashStr) in set(map(tuple, ModuleHashPairList)): ModuleHashPairList.insert(0, (PreMakeHashStr, MakeHashStr)) with open(ModuleHashPair, 'w') as f: json.dump(ModuleHashPairList, f, indent=2) # Copy files to Cache destination dirs if not self.OutputFile: Ma = self.BuildDatabase[self.MetaFile, self.Arch, self.BuildTarget, self.ToolChain] self.OutputFile = Ma.Binaries for File in self.OutputFile: if File.startswith(os.path.abspath(self.FfsOutputDir)+os.sep): self.CacheCopyFile(CacheFfsDir, self.FfsOutputDir, File) else: if self.Name + ".autogen.hash." in File or \ self.Name + ".autogen.hashchain." in File or \ self.Name + ".hash." in File or \ self.Name + ".hashchain." in File or \ self.Name + ".PreMakeHashFileList." in File or \ self.Name + ".MakeHashFileList." in File: self.CacheCopyFile(FileDir, self.BuildDir, File) else: self.CacheCopyFile(CacheFileDir, self.BuildDir, File) ## Create makefile for the module and its dependent libraries # # @param CreateLibraryMakeFile Flag indicating if or not the makefiles of # dependent libraries will be created # @cached_class_function def CreateMakeFile(self, CreateLibraryMakeFile=True, GenFfsList = []): # nest this function inside it's only caller. def CreateTimeStamp(): FileSet = {self.MetaFile.Path} for SourceFile in self.Module.Sources: FileSet.add (SourceFile.Path) for Lib in self.DependentLibraryList: FileSet.add (Lib.MetaFile.Path) for f in self.AutoGenDepSet: FileSet.add (f.Path) if os.path.exists (self.TimeStampPath): os.remove (self.TimeStampPath) SaveFileOnChange(self.TimeStampPath, "\n".join(FileSet), False) # Ignore generating makefile when it is a binary module if self.IsBinaryModule: return self.GenFfsList = GenFfsList if not self.IsLibrary and CreateLibraryMakeFile: for LibraryAutoGen in self.LibraryAutoGenList: LibraryAutoGen.CreateMakeFile() # CanSkip uses timestamps to determine build skipping if self.CanSkip(): return if len(self.CustomMakefile) == 0: Makefile = GenMake.ModuleMakefile(self) else: Makefile = GenMake.CustomMakefile(self) if Makefile.Generate(): EdkLogger.debug(EdkLogger.DEBUG_9, "Generated makefile for module %s [%s]" % (self.Name, self.Arch)) else: EdkLogger.debug(EdkLogger.DEBUG_9, "Skipped the generation of makefile for module %s [%s]" % (self.Name, self.Arch)) CreateTimeStamp() MakefileType = Makefile._FileType MakefileName = Makefile._FILE_NAME_[MakefileType] MakefilePath = os.path.join(self.MakeFileDir, MakefileName) FilePath = path.join(self.BuildDir, self.Name + ".makefile") SaveFileOnChange(FilePath, MakefilePath, False) def CopyBinaryFiles(self): for File in self.Module.Binaries: SrcPath = File.Path DstPath = os.path.join(self.OutputDir, os.path.basename(SrcPath)) CopyLongFilePath(SrcPath, DstPath) ## Create autogen code for the module and its dependent libraries # # @param CreateLibraryCodeFile Flag indicating if or not the code of # dependent libraries will be created # def CreateCodeFile(self, CreateLibraryCodeFile=True): if self.IsCodeFileCreated: return # Need to generate PcdDatabase even PcdDriver is binarymodule if self.IsBinaryModule and self.PcdIsDriver != '': CreatePcdDatabaseCode(self, TemplateString(), TemplateString()) return if self.IsBinaryModule: if self.IsLibrary: self.CopyBinaryFiles() return if not self.IsLibrary and CreateLibraryCodeFile: for LibraryAutoGen in self.LibraryAutoGenList: LibraryAutoGen.CreateCodeFile() self.LibraryAutoGenList AutoGenList = [] IgoredAutoGenList = [] for File in self.AutoGenFileList: if GenC.Generate(File.Path, self.AutoGenFileList[File], File.IsBinary): AutoGenList.append(str(File)) else: IgoredAutoGenList.append(str(File)) for ModuleType in self.DepexList: # Ignore empty [depex] section or [depex] section for SUP_MODULE_USER_DEFINED module if len(self.DepexList[ModuleType]) == 0 or ModuleType == SUP_MODULE_USER_DEFINED or ModuleType == SUP_MODULE_HOST_APPLICATION: continue Dpx = GenDepex.DependencyExpression(self.DepexList[ModuleType], ModuleType, True) DpxFile = gAutoGenDepexFileName % {"module_name" : self.Name} if len(Dpx.PostfixNotation) != 0: self.DepexGenerated = True if Dpx.Generate(path.join(self.OutputDir, DpxFile)): AutoGenList.append(str(DpxFile)) else: IgoredAutoGenList.append(str(DpxFile)) if IgoredAutoGenList == []: EdkLogger.debug(EdkLogger.DEBUG_9, "Generated [%s] files for module %s [%s]" % (" ".join(AutoGenList), self.Name, self.Arch)) elif AutoGenList == []: EdkLogger.debug(EdkLogger.DEBUG_9, "Skipped the generation of [%s] files for module %s [%s]" % (" ".join(IgoredAutoGenList), self.Name, self.Arch)) else: EdkLogger.debug(EdkLogger.DEBUG_9, "Generated [%s] (skipped %s) files for module %s [%s]" % (" ".join(AutoGenList), " ".join(IgoredAutoGenList), self.Name, self.Arch)) self.IsCodeFileCreated = True return AutoGenList ## Summarize the ModuleAutoGen objects of all libraries used by this module @cached_property def LibraryAutoGenList(self): RetVal = [] for Library in self.DependentLibraryList: La = ModuleAutoGen( self.Workspace, Library.MetaFile, self.BuildTarget, self.ToolChain, self.Arch, self.PlatformInfo.MetaFile, self.DataPipe ) La.IsLibrary = True if La not in RetVal: RetVal.append(La) for Lib in La.CodaTargetList: self._ApplyBuildRule(Lib.Target, TAB_UNKNOWN_FILE) return RetVal def GenCMakeHash(self): # GenCMakeHash can only be called in --binary-destination # Never called in multiprocessing and always directly save result in main process, # so no need remote dict to share the gCMakeHashFile result with main process DependencyFileSet = set() # Add AutoGen files if self.AutoGenFileList: for File in set(self.AutoGenFileList): DependencyFileSet.add(File) # Add Makefile abspath = path.join(self.BuildDir, self.Name + ".makefile") try: with open(LongFilePath(abspath),"r") as fd: lines = fd.readlines() except Exception as e: EdkLogger.error("build",FILE_NOT_FOUND, "%s doesn't exist" % abspath, ExtraData=str(e), RaiseError=False) if lines: DependencyFileSet.update(lines) # Caculate all above dependency files hash # Initialze hash object FileList = [] m = hashlib.md5() for File in sorted(DependencyFileSet, key=lambda x: str(x)): if not path.exists(LongFilePath(str(File))): EdkLogger.quiet("[cache warning]: header file %s is missing for module: %s[%s]" % (File, self.MetaFile.Path, self.Arch)) continue with open(LongFilePath(str(File)), 'rb') as f: Content = f.read() m.update(Content) FileList.append((str(File), hashlib.md5(Content).hexdigest())) HashChainFile = path.join(self.BuildDir, self.Name + ".autogen.hashchain." + m.hexdigest()) GlobalData.gCMakeHashFile[(self.MetaFile.Path, self.Arch)] = HashChainFile try: with open(LongFilePath(HashChainFile), 'w') as f: json.dump(FileList, f, indent=2) except: EdkLogger.quiet("[cache warning]: fail to save hashchain file:%s" % HashChainFile) return False def GenModuleHash(self): # GenModuleHash only called after autogen phase # Never called in multiprocessing and always directly save result in main process, # so no need remote dict to share the gModuleHashFile result with main process # # GenPreMakefileHashList consume no dict. # GenPreMakefileHashList produce local gModuleHashFile dict. DependencyFileSet = set() # Add Module Meta file DependencyFileSet.add(self.MetaFile.Path) # Add Module's source files if self.SourceFileList: for File in set(self.SourceFileList): DependencyFileSet.add(File.Path) # Add modules's include header files # Directly use the deps.txt file in the module BuildDir abspath = path.join(self.BuildDir, "deps.txt") rt = None try: with open(LongFilePath(abspath),"r") as fd: lines = fd.readlines() if lines: rt = set([item.lstrip().strip("\n") for item in lines if item.strip("\n").endswith(".h")]) except Exception as e: EdkLogger.error("build",FILE_NOT_FOUND, "%s doesn't exist" % abspath, ExtraData=str(e), RaiseError=False) if rt: DependencyFileSet.update(rt) # Caculate all above dependency files hash # Initialze hash object FileList = [] m = hashlib.md5() BuildDirStr = path.abspath(self.BuildDir).lower() for File in sorted(DependencyFileSet, key=lambda x: str(x)): # Skip the AutoGen files in BuildDir which already been # included in .autogen.hash. file if BuildDirStr in path.abspath(File).lower(): continue if not path.exists(LongFilePath(File)): EdkLogger.quiet("[cache warning]: header file %s is missing for module: %s[%s]" % (File, self.MetaFile.Path, self.Arch)) continue with open(LongFilePath(File), 'rb') as f: Content = f.read() m.update(Content) FileList.append((File, hashlib.md5(Content).hexdigest())) HashChainFile = path.join(self.BuildDir, self.Name + ".hashchain." + m.hexdigest()) GlobalData.gModuleHashFile[(self.MetaFile.Path, self.Arch)] = HashChainFile try: with open(LongFilePath(HashChainFile), 'w') as f: json.dump(FileList, f, indent=2) except: EdkLogger.quiet("[cache warning]: fail to save hashchain file:%s" % HashChainFile) return False def GenPreMakefileHashList(self): # GenPreMakefileHashList consume below dicts: # gPlatformHashFile # gPackageHashFile # gModuleHashFile # GenPreMakefileHashList produce no dict. # gModuleHashFile items might be produced in multiprocessing, so # need check gModuleHashFile remote dict # skip binary module if self.IsBinaryModule: return FileList = [] m = hashlib.md5() # Add Platform level hash HashFile = GlobalData.gPlatformHashFile if path.exists(LongFilePath(HashFile)): FileList.append(HashFile) m.update(HashFile.encode('utf-8')) else: EdkLogger.quiet("[cache warning]: No Platform HashFile: %s" % HashFile) # Add Package level hash if self.DependentPackageList: for Pkg in sorted(self.DependentPackageList, key=lambda x: x.PackageName): if not (Pkg.PackageName, Pkg.Arch) in GlobalData.gPackageHashFile: EdkLogger.quiet("[cache warning]:No Package %s for module %s[%s]" % (Pkg.PackageName, self.MetaFile.Path, self.Arch)) continue HashFile = GlobalData.gPackageHashFile[(Pkg.PackageName, Pkg.Arch)] if path.exists(LongFilePath(HashFile)): FileList.append(HashFile) m.update(HashFile.encode('utf-8')) else: EdkLogger.quiet("[cache warning]:No Package HashFile: %s" % HashFile) # Add Module self # GenPreMakefileHashList needed in both --binary-destination # and --hash. And --hash might save ModuleHashFile in remote dict # during multiprocessing. if (self.MetaFile.Path, self.Arch) in GlobalData.gModuleHashFile: HashFile = GlobalData.gModuleHashFile[(self.MetaFile.Path, self.Arch)] else: EdkLogger.quiet("[cache error]:No ModuleHashFile for module: %s[%s]" % (self.MetaFile.Path, self.Arch)) if path.exists(LongFilePath(HashFile)): FileList.append(HashFile) m.update(HashFile.encode('utf-8')) else: EdkLogger.quiet("[cache warning]:No Module HashFile: %s" % HashFile) # Add Library hash if self.LibraryAutoGenList: for Lib in sorted(self.LibraryAutoGenList, key=lambda x: x.MetaFile.Path): if (Lib.MetaFile.Path, Lib.Arch) in GlobalData.gModuleHashFile: HashFile = GlobalData.gModuleHashFile[(Lib.MetaFile.Path, Lib.Arch)] else: EdkLogger.quiet("[cache error]:No ModuleHashFile for lib: %s[%s]" % (Lib.MetaFile.Path, Lib.Arch)) if path.exists(LongFilePath(HashFile)): FileList.append(HashFile) m.update(HashFile.encode('utf-8')) else: EdkLogger.quiet("[cache warning]:No Lib HashFile: %s" % HashFile) # Save PreMakeHashFileList FilePath = path.join(self.BuildDir, self.Name + ".PreMakeHashFileList." + m.hexdigest()) try: with open(LongFilePath(FilePath), 'w') as f: json.dump(FileList, f, indent=0) except: EdkLogger.quiet("[cache warning]: fail to save PreMake HashFileList: %s" % FilePath) def GenMakefileHashList(self): # GenMakefileHashList only need in --binary-destination which will # everything in local dict. So don't need check remote dict. # skip binary module if self.IsBinaryModule: return FileList = [] m = hashlib.md5() # Add AutoGen hash HashFile = GlobalData.gCMakeHashFile[(self.MetaFile.Path, self.Arch)] if path.exists(LongFilePath(HashFile)): FileList.append(HashFile) m.update(HashFile.encode('utf-8')) else: EdkLogger.quiet("[cache warning]:No AutoGen HashFile: %s" % HashFile) # Add Module self if (self.MetaFile.Path, self.Arch) in GlobalData.gModuleHashFile: HashFile = GlobalData.gModuleHashFile[(self.MetaFile.Path, self.Arch)] else: EdkLogger.quiet("[cache error]:No ModuleHashFile for module: %s[%s]" % (self.MetaFile.Path, self.Arch)) if path.exists(LongFilePath(HashFile)): FileList.append(HashFile) m.update(HashFile.encode('utf-8')) else: EdkLogger.quiet("[cache warning]:No Module HashFile: %s" % HashFile) # Add Library hash if self.LibraryAutoGenList: for Lib in sorted(self.LibraryAutoGenList, key=lambda x: x.MetaFile.Path): if (Lib.MetaFile.Path, Lib.Arch) in GlobalData.gModuleHashFile: HashFile = GlobalData.gModuleHashFile[(Lib.MetaFile.Path, Lib.Arch)] else: EdkLogger.quiet("[cache error]:No ModuleHashFile for lib: %s[%s]" % (Lib.MetaFile.Path, Lib.Arch)) if path.exists(LongFilePath(HashFile)): FileList.append(HashFile) m.update(HashFile.encode('utf-8')) else: EdkLogger.quiet("[cache warning]:No Lib HashFile: %s" % HashFile) # Save MakeHashFileList FilePath = path.join(self.BuildDir, self.Name + ".MakeHashFileList." + m.hexdigest()) try: with open(LongFilePath(FilePath), 'w') as f: json.dump(FileList, f, indent=0) except: EdkLogger.quiet("[cache warning]: fail to save Make HashFileList: %s" % FilePath) def CheckHashChainFile(self, HashChainFile): # Assume the HashChainFile basename format is the 'x.hashchain.16BytesHexStr' # The x is module name and the 16BytesHexStr is md5 hexdigest of # all hashchain files content HashStr = HashChainFile.split('.')[-1] if len(HashStr) != 32: EdkLogger.quiet("[cache error]: wrong format HashChainFile:%s" % (File)) return False try: with open(LongFilePath(HashChainFile), 'r') as f: HashChainList = json.load(f) except: EdkLogger.quiet("[cache error]: fail to load HashChainFile: %s" % HashChainFile) return False # Print the different file info # print(HashChainFile) for idx, (SrcFile, SrcHash) in enumerate (HashChainList): if SrcFile in GlobalData.gFileHashDict: DestHash = GlobalData.gFileHashDict[SrcFile] else: try: with open(LongFilePath(SrcFile), 'rb') as f: Content = f.read() DestHash = hashlib.md5(Content).hexdigest() GlobalData.gFileHashDict[SrcFile] = DestHash except IOError as X: # cache miss if SrcFile is removed in new version code GlobalData.gFileHashDict[SrcFile] = 0 EdkLogger.quiet("[cache insight]: first cache miss file in %s is %s" % (HashChainFile, SrcFile)) return False if SrcHash != DestHash: EdkLogger.quiet("[cache insight]: first cache miss file in %s is %s" % (HashChainFile, SrcFile)) return False return True ## Decide whether we can skip the left autogen and make process def CanSkipbyMakeCache(self): # For --binary-source only # CanSkipbyMakeCache consume below dicts: # gModuleMakeCacheStatus # gHashChainStatus # GenPreMakefileHashList produce gModuleMakeCacheStatus, gModuleHashFile dict. # all these dicts might be produced in multiprocessing, so # need check these remote dict if not GlobalData.gBinCacheSource: return False if (self.MetaFile.Path, self.Arch) in GlobalData.gModuleMakeCacheStatus: return GlobalData.gModuleMakeCacheStatus[(self.MetaFile.Path, self.Arch)] # If Module is binary, which has special build rule, do not skip by cache. if self.IsBinaryModule: print("[cache miss]: MakeCache: Skip BinaryModule:", self.MetaFile.Path, self.Arch) GlobalData.gModuleMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = False return False # see .inc as binary file, do not skip by hash for f_ext in self.SourceFileList: if '.inc' in str(f_ext): print("[cache miss]: MakeCache: Skip '.inc' File:", self.MetaFile.Path, self.Arch) GlobalData.gModuleMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = False return False ModuleCacheDir = path.join(GlobalData.gBinCacheSource, self.PlatformInfo.OutputDir, self.BuildTarget + "_" + self.ToolChain, self.Arch, self.SourceDir, self.MetaFile.BaseName) FfsDir = path.join(GlobalData.gBinCacheSource, self.PlatformInfo.OutputDir, self.BuildTarget + "_" + self.ToolChain, TAB_FV_DIRECTORY, "Ffs", self.Guid + self.Name) ModuleHashPairList = [] # tuple list: [tuple(PreMakefileHash, MakeHash)] ModuleHashPair = path.join(ModuleCacheDir, self.Name + ".ModuleHashPair") try: with open(LongFilePath(ModuleHashPair), 'r') as f: ModuleHashPairList = json.load(f) except: # ModuleHashPair might not exist for new added module GlobalData.gModuleMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = False EdkLogger.quiet("[cache warning]: fail to load ModuleHashPair file: %s" % ModuleHashPair) print("[cache miss]: MakeCache:", self.MetaFile.Path, self.Arch) return False # Check the PreMakeHash in ModuleHashPairList one by one for idx, (PreMakefileHash, MakeHash) in enumerate (ModuleHashPairList): SourceHashDir = path.join(ModuleCacheDir, MakeHash) SourceFfsHashDir = path.join(FfsDir, MakeHash) PreMakeHashFileList_FilePah = path.join(ModuleCacheDir, self.Name + ".PreMakeHashFileList." + PreMakefileHash) MakeHashFileList_FilePah = path.join(ModuleCacheDir, self.Name + ".MakeHashFileList." + MakeHash) try: with open(LongFilePath(MakeHashFileList_FilePah), 'r') as f: MakeHashFileList = json.load(f) except: EdkLogger.quiet("[cache error]: fail to load MakeHashFileList file: %s" % MakeHashFileList_FilePah) continue HashMiss = False for HashChainFile in MakeHashFileList: HashChainStatus = None if HashChainFile in GlobalData.gHashChainStatus: HashChainStatus = GlobalData.gHashChainStatus[HashChainFile] if HashChainStatus == False: HashMiss = True break elif HashChainStatus == True: continue # Convert to path start with cache source dir RelativePath = os.path.relpath(HashChainFile, self.WorkspaceDir) NewFilePath = os.path.join(GlobalData.gBinCacheSource, RelativePath) if self.CheckHashChainFile(NewFilePath): GlobalData.gHashChainStatus[HashChainFile] = True # Save the module self HashFile for GenPreMakefileHashList later usage if self.Name + ".hashchain." in HashChainFile: GlobalData.gModuleHashFile[(self.MetaFile.Path, self.Arch)] = HashChainFile else: GlobalData.gHashChainStatus[HashChainFile] = False HashMiss = True break if HashMiss: continue # PreMakefile cache hit, restore the module build result for root, dir, files in os.walk(SourceHashDir): for f in files: File = path.join(root, f) self.CacheCopyFile(self.BuildDir, SourceHashDir, File) if os.path.exists(SourceFfsHashDir): for root, dir, files in os.walk(SourceFfsHashDir): for f in files: File = path.join(root, f) self.CacheCopyFile(self.FfsOutputDir, SourceFfsHashDir, File) if self.Name == "PcdPeim" or self.Name == "PcdDxe": CreatePcdDatabaseCode(self, TemplateString(), TemplateString()) print("[cache hit]: MakeCache:", self.MetaFile.Path, self.Arch) GlobalData.gModuleMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = True return True print("[cache miss]: MakeCache:", self.MetaFile.Path, self.Arch) GlobalData.gModuleMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = False return False ## Decide whether we can skip the left autogen and make process def CanSkipbyPreMakeCache(self): # CanSkipbyPreMakeCache consume below dicts: # gModulePreMakeCacheStatus # gHashChainStatus # gModuleHashFile # GenPreMakefileHashList produce gModulePreMakeCacheStatus dict. # all these dicts might be produced in multiprocessing, so # need check these remote dicts if not GlobalData.gUseHashCache or GlobalData.gBinCacheDest: return False if (self.MetaFile.Path, self.Arch) in GlobalData.gModulePreMakeCacheStatus: return GlobalData.gModulePreMakeCacheStatus[(self.MetaFile.Path, self.Arch)] # If Module is binary, which has special build rule, do not skip by cache. if self.IsBinaryModule: print("[cache miss]: PreMakeCache: Skip BinaryModule:", self.MetaFile.Path, self.Arch) GlobalData.gModulePreMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = False return False # see .inc as binary file, do not skip by hash for f_ext in self.SourceFileList: if '.inc' in str(f_ext): print("[cache miss]: PreMakeCache: Skip '.inc' File:", self.MetaFile.Path, self.Arch) GlobalData.gModulePreMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = False return False # For --hash only in the incremental build if not GlobalData.gBinCacheSource: Files = [path.join(self.BuildDir, f) for f in os.listdir(self.BuildDir) if path.isfile(path.join(self.BuildDir, f))] PreMakeHashFileList_FilePah = None MakeTimeStamp = 0 # Find latest PreMakeHashFileList file in self.BuildDir folder for File in Files: if ".PreMakeHashFileList." in File: FileTimeStamp = os.stat(path.join(self.BuildDir, File))[8] if FileTimeStamp > MakeTimeStamp: MakeTimeStamp = FileTimeStamp PreMakeHashFileList_FilePah = File if not PreMakeHashFileList_FilePah: GlobalData.gModulePreMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = False return False try: with open(LongFilePath(PreMakeHashFileList_FilePah), 'r') as f: PreMakeHashFileList = json.load(f) except: EdkLogger.quiet("[cache error]: fail to load PreMakeHashFileList file: %s" % PreMakeHashFileList_FilePah) print("[cache miss]: PreMakeCache:", self.MetaFile.Path, self.Arch) GlobalData.gModulePreMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = False return False HashMiss = False for HashChainFile in PreMakeHashFileList: HashChainStatus = None if HashChainFile in GlobalData.gHashChainStatus: HashChainStatus = GlobalData.gHashChainStatus[HashChainFile] if HashChainStatus == False: HashMiss = True break elif HashChainStatus == True: continue if self.CheckHashChainFile(HashChainFile): GlobalData.gHashChainStatus[HashChainFile] = True # Save the module self HashFile for GenPreMakefileHashList later usage if self.Name + ".hashchain." in HashChainFile: GlobalData.gModuleHashFile[(self.MetaFile.Path, self.Arch)] = HashChainFile else: GlobalData.gHashChainStatus[HashChainFile] = False HashMiss = True break if HashMiss: print("[cache miss]: PreMakeCache:", self.MetaFile.Path, self.Arch) GlobalData.gModulePreMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = False return False else: print("[cache hit]: PreMakeCache:", self.MetaFile.Path, self.Arch) GlobalData.gModulePreMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = True return True ModuleCacheDir = path.join(GlobalData.gBinCacheSource, self.PlatformInfo.OutputDir, self.BuildTarget + "_" + self.ToolChain, self.Arch, self.SourceDir, self.MetaFile.BaseName) FfsDir = path.join(GlobalData.gBinCacheSource, self.PlatformInfo.OutputDir, self.BuildTarget + "_" + self.ToolChain, TAB_FV_DIRECTORY, "Ffs", self.Guid + self.Name) ModuleHashPairList = [] # tuple list: [tuple(PreMakefileHash, MakeHash)] ModuleHashPair = path.join(ModuleCacheDir, self.Name + ".ModuleHashPair") try: with open(LongFilePath(ModuleHashPair), 'r') as f: ModuleHashPairList = json.load(f) except: # ModuleHashPair might not exist for new added module GlobalData.gModulePreMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = False EdkLogger.quiet("[cache warning]: fail to load ModuleHashPair file: %s" % ModuleHashPair) print("[cache miss]: PreMakeCache:", self.MetaFile.Path, self.Arch) return False # Check the PreMakeHash in ModuleHashPairList one by one for idx, (PreMakefileHash, MakeHash) in enumerate (ModuleHashPairList): SourceHashDir = path.join(ModuleCacheDir, MakeHash) SourceFfsHashDir = path.join(FfsDir, MakeHash) PreMakeHashFileList_FilePah = path.join(ModuleCacheDir, self.Name + ".PreMakeHashFileList." + PreMakefileHash) MakeHashFileList_FilePah = path.join(ModuleCacheDir, self.Name + ".MakeHashFileList." + MakeHash) try: with open(LongFilePath(PreMakeHashFileList_FilePah), 'r') as f: PreMakeHashFileList = json.load(f) except: EdkLogger.quiet("[cache error]: fail to load PreMakeHashFileList file: %s" % PreMakeHashFileList_FilePah) continue HashMiss = False for HashChainFile in PreMakeHashFileList: HashChainStatus = None if HashChainFile in GlobalData.gHashChainStatus: HashChainStatus = GlobalData.gHashChainStatus[HashChainFile] if HashChainStatus == False: HashMiss = True break elif HashChainStatus == True: continue # Convert to path start with cache source dir RelativePath = os.path.relpath(HashChainFile, self.WorkspaceDir) NewFilePath = os.path.join(GlobalData.gBinCacheSource, RelativePath) if self.CheckHashChainFile(NewFilePath): GlobalData.gHashChainStatus[HashChainFile] = True else: GlobalData.gHashChainStatus[HashChainFile] = False HashMiss = True break if HashMiss: continue # PreMakefile cache hit, restore the module build result for root, dir, files in os.walk(SourceHashDir): for f in files: File = path.join(root, f) self.CacheCopyFile(self.BuildDir, SourceHashDir, File) if os.path.exists(SourceFfsHashDir): for root, dir, files in os.walk(SourceFfsHashDir): for f in files: File = path.join(root, f) self.CacheCopyFile(self.FfsOutputDir, SourceFfsHashDir, File) if self.Name == "PcdPeim" or self.Name == "PcdDxe": CreatePcdDatabaseCode(self, TemplateString(), TemplateString()) print("[cache hit]: PreMakeCache:", self.MetaFile.Path, self.Arch) GlobalData.gModulePreMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = True return True print("[cache miss]: PreMakeCache:", self.MetaFile.Path, self.Arch) GlobalData.gModulePreMakeCacheStatus[(self.MetaFile.Path, self.Arch)] = False return False ## Decide whether we can skip the Module build def CanSkipbyCache(self, gHitSet): # Hashing feature is off if not GlobalData.gBinCacheSource: return False if self in gHitSet: return True return False ## Decide whether we can skip the ModuleAutoGen process # If any source file is newer than the module than we cannot skip # def CanSkip(self): # Don't skip if cache feature enabled if GlobalData.gUseHashCache or GlobalData.gBinCacheDest or GlobalData.gBinCacheSource: return False if self.MakeFileDir in GlobalData.gSikpAutoGenCache: return True if not os.path.exists(self.TimeStampPath): return False #last creation time of the module DstTimeStamp = os.stat(self.TimeStampPath)[8] SrcTimeStamp = self.Workspace._SrcTimeStamp if SrcTimeStamp > DstTimeStamp: return False with open(self.TimeStampPath,'r') as f: for source in f: source = source.rstrip('\n') if not os.path.exists(source): return False if source not in ModuleAutoGen.TimeDict : ModuleAutoGen.TimeDict[source] = os.stat(source)[8] if ModuleAutoGen.TimeDict[source] > DstTimeStamp: return False GlobalData.gSikpAutoGenCache.add(self.MakeFileDir) return True @cached_property def TimeStampPath(self): return os.path.join(self.MakeFileDir, 'AutoGenTimeStamp')