## @file # Create makefile for MS nmake and GNU make # # Copyright (c) 2019, Intel Corporation. All rights reserved.
# SPDX-License-Identifier: BSD-2-Clause-Patent # from __future__ import absolute_import import multiprocessing as mp import threading from Common.Misc import PathClass from AutoGen.ModuleAutoGen import ModuleAutoGen from AutoGen.ModuleAutoGenHelper import WorkSpaceInfo,AutoGenInfo import Common.GlobalData as GlobalData import Common.EdkLogger as EdkLogger import os from Common.MultipleWorkspace import MultipleWorkspace as mws from AutoGen.AutoGen import AutoGen from Workspace.WorkspaceDatabase import BuildDB try: from queue import Empty except: from Queue import Empty import traceback import sys from AutoGen.DataPipe import MemoryDataPipe import logging import time def clearQ(q): try: while True: q.get_nowait() except Empty: pass class LogAgent(threading.Thread): def __init__(self,log_q,log_level,log_file=None): super(LogAgent,self).__init__() self.log_q = log_q self.log_level = log_level self.log_file = log_file def InitLogger(self): # For DEBUG level (All DEBUG_0~9 are applicable) self._DebugLogger_agent = logging.getLogger("tool_debug_agent") _DebugFormatter = logging.Formatter("[%(asctime)s.%(msecs)d]: %(message)s", datefmt="%H:%M:%S") self._DebugLogger_agent.setLevel(self.log_level) _DebugChannel = logging.StreamHandler(sys.stdout) _DebugChannel.setFormatter(_DebugFormatter) self._DebugLogger_agent.addHandler(_DebugChannel) # For VERBOSE, INFO, WARN level self._InfoLogger_agent = logging.getLogger("tool_info_agent") _InfoFormatter = logging.Formatter("%(message)s") self._InfoLogger_agent.setLevel(self.log_level) _InfoChannel = logging.StreamHandler(sys.stdout) _InfoChannel.setFormatter(_InfoFormatter) self._InfoLogger_agent.addHandler(_InfoChannel) # For ERROR level self._ErrorLogger_agent = logging.getLogger("tool_error_agent") _ErrorFormatter = logging.Formatter("%(message)s") self._ErrorLogger_agent.setLevel(self.log_level) _ErrorCh = logging.StreamHandler(sys.stderr) _ErrorCh.setFormatter(_ErrorFormatter) self._ErrorLogger_agent.addHandler(_ErrorCh) if self.log_file: if os.path.exists(self.log_file): os.remove(self.log_file) _Ch = logging.FileHandler(self.log_file) _Ch.setFormatter(_DebugFormatter) self._DebugLogger_agent.addHandler(_Ch) _Ch= logging.FileHandler(self.log_file) _Ch.setFormatter(_InfoFormatter) self._InfoLogger_agent.addHandler(_Ch) _Ch = logging.FileHandler(self.log_file) _Ch.setFormatter(_ErrorFormatter) self._ErrorLogger_agent.addHandler(_Ch) def run(self): self.InitLogger() while True: log_message = self.log_q.get() if log_message is None: break if log_message.name == "tool_error": self._ErrorLogger_agent.log(log_message.levelno,log_message.getMessage()) elif log_message.name == "tool_info": self._InfoLogger_agent.log(log_message.levelno,log_message.getMessage()) elif log_message.name == "tool_debug": self._DebugLogger_agent.log(log_message.levelno,log_message.getMessage()) else: self._InfoLogger_agent.log(log_message.levelno,log_message.getMessage()) def kill(self): self.log_q.put(None) class AutoGenManager(threading.Thread): def __init__(self,autogen_workers, feedback_q,error_event): super(AutoGenManager,self).__init__() self.autogen_workers = autogen_workers self.feedback_q = feedback_q self.Status = True self.error_event = error_event def run(self): try: fin_num = 0 while True: badnews = self.feedback_q.get() if badnews is None: break if badnews == "Done": fin_num += 1 elif badnews == "QueueEmpty": EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), badnews)) self.TerminateWorkers() else: EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), badnews)) self.Status = False self.TerminateWorkers() if fin_num == len(self.autogen_workers): self.clearQueue() for w in self.autogen_workers: w.join() break except Exception: return def clearQueue(self): taskq = self.autogen_workers[0].module_queue logq = self.autogen_workers[0].log_q clearQ(taskq) clearQ(self.feedback_q) clearQ(logq) # Copy the cache queue itmes to parent thread before clear cacheq = self.autogen_workers[0].cache_q try: cache_num = 0 while True: item = cacheq.get() if item == "CacheDone": cache_num += 1 else: GlobalData.gModuleAllCacheStatus.add(item) if cache_num == len(self.autogen_workers): break except: print ("cache_q error") def TerminateWorkers(self): self.error_event.set() def kill(self): self.feedback_q.put(None) class AutoGenWorkerInProcess(mp.Process): def __init__(self,module_queue,data_pipe_file_path,feedback_q,file_lock,cache_q,log_q,error_event): mp.Process.__init__(self) self.module_queue = module_queue self.data_pipe_file_path =data_pipe_file_path self.data_pipe = None self.feedback_q = feedback_q self.PlatformMetaFileSet = {} self.file_lock = file_lock self.cache_q = cache_q self.log_q = log_q self.error_event = error_event def GetPlatformMetaFile(self,filepath,root): try: return self.PlatformMetaFileSet[(filepath,root)] except: self.PlatformMetaFileSet[(filepath,root)] = filepath return self.PlatformMetaFileSet[(filepath,root)] def run(self): try: taskname = "Init" with self.file_lock: try: self.data_pipe = MemoryDataPipe() self.data_pipe.load(self.data_pipe_file_path) except: self.feedback_q.put(taskname + ":" + "load data pipe %s failed." % self.data_pipe_file_path) EdkLogger.LogClientInitialize(self.log_q) loglevel = self.data_pipe.Get("LogLevel") if not loglevel: loglevel = EdkLogger.INFO EdkLogger.SetLevel(loglevel) target = self.data_pipe.Get("P_Info").get("Target") toolchain = self.data_pipe.Get("P_Info").get("ToolChain") archlist = self.data_pipe.Get("P_Info").get("ArchList") active_p = self.data_pipe.Get("P_Info").get("ActivePlatform") workspacedir = self.data_pipe.Get("P_Info").get("WorkspaceDir") PackagesPath = os.getenv("PACKAGES_PATH") mws.setWs(workspacedir, PackagesPath) self.Wa = WorkSpaceInfo( workspacedir,active_p,target,toolchain,archlist ) self.Wa._SrcTimeStamp = self.data_pipe.Get("Workspace_timestamp") GlobalData.gGlobalDefines = self.data_pipe.Get("G_defines") GlobalData.gCommandLineDefines = self.data_pipe.Get("CL_defines") GlobalData.gCommandMaxLength = self.data_pipe.Get('gCommandMaxLength') os.environ._data = self.data_pipe.Get("Env_Var") GlobalData.gWorkspace = workspacedir GlobalData.gDisableIncludePathCheck = False GlobalData.gFdfParser = self.data_pipe.Get("FdfParser") GlobalData.gDatabasePath = self.data_pipe.Get("DatabasePath") GlobalData.gUseHashCache = self.data_pipe.Get("UseHashCache") GlobalData.gBinCacheSource = self.data_pipe.Get("BinCacheSource") GlobalData.gBinCacheDest = self.data_pipe.Get("BinCacheDest") GlobalData.gPlatformHashFile = self.data_pipe.Get("PlatformHashFile") GlobalData.gModulePreMakeCacheStatus = dict() GlobalData.gModuleMakeCacheStatus = dict() GlobalData.gHashChainStatus = dict() GlobalData.gCMakeHashFile = dict() GlobalData.gModuleHashFile = dict() GlobalData.gFileHashDict = dict() GlobalData.gEnableGenfdsMultiThread = self.data_pipe.Get("EnableGenfdsMultiThread") GlobalData.gPlatformFinalPcds = self.data_pipe.Get("gPlatformFinalPcds") GlobalData.file_lock = self.file_lock CommandTarget = self.data_pipe.Get("CommandTarget") pcd_from_build_option = [] for pcd_tuple in self.data_pipe.Get("BuildOptPcd"): pcd_id = ".".join((pcd_tuple[0],pcd_tuple[1])) if pcd_tuple[2].strip(): pcd_id = ".".join((pcd_id,pcd_tuple[2])) pcd_from_build_option.append("=".join((pcd_id,pcd_tuple[3]))) GlobalData.BuildOptionPcd = pcd_from_build_option module_count = 0 FfsCmd = self.data_pipe.Get("FfsCommand") if FfsCmd is None: FfsCmd = {} GlobalData.FfsCmd = FfsCmd PlatformMetaFile = self.GetPlatformMetaFile(self.data_pipe.Get("P_Info").get("ActivePlatform"), self.data_pipe.Get("P_Info").get("WorkspaceDir")) while True: if self.error_event.is_set(): break module_count += 1 try: module_file,module_root,module_path,module_basename,module_originalpath,module_arch,IsLib = self.module_queue.get_nowait() except Empty: EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), "Fake Empty.")) time.sleep(0.01) continue if module_file is None: EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), "Worker get the last item in the queue.")) self.feedback_q.put("QueueEmpty") time.sleep(0.01) continue modulefullpath = os.path.join(module_root,module_file) taskname = " : ".join((modulefullpath,module_arch)) module_metafile = PathClass(module_file,module_root) if module_path: module_metafile.Path = module_path if module_basename: module_metafile.BaseName = module_basename if module_originalpath: module_metafile.OriginalPath = PathClass(module_originalpath,module_root) arch = module_arch target = self.data_pipe.Get("P_Info").get("Target") toolchain = self.data_pipe.Get("P_Info").get("ToolChain") Ma = ModuleAutoGen(self.Wa,module_metafile,target,toolchain,arch,PlatformMetaFile,self.data_pipe) Ma.IsLibrary = IsLib # SourceFileList calling sequence impact the makefile string sequence. # Create cached SourceFileList here to unify its calling sequence for both # CanSkipbyPreMakeCache and CreateCodeFile/CreateMakeFile. RetVal = Ma.SourceFileList if GlobalData.gUseHashCache and not GlobalData.gBinCacheDest and CommandTarget in [None, "", "all"]: try: CacheResult = Ma.CanSkipbyPreMakeCache() except: CacheResult = False self.feedback_q.put(taskname) if CacheResult: self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "PreMakeCache", True)) continue else: self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "PreMakeCache", False)) Ma.CreateCodeFile(False) Ma.CreateMakeFile(False,GenFfsList=FfsCmd.get((Ma.MetaFile.Path, Ma.Arch),[])) Ma.CreateAsBuiltInf() if GlobalData.gBinCacheSource and CommandTarget in [None, "", "all"]: try: CacheResult = Ma.CanSkipbyMakeCache() except: CacheResult = False self.feedback_q.put(taskname) if CacheResult: self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "MakeCache", True)) continue else: self.cache_q.put((Ma.MetaFile.Path, Ma.Arch, "MakeCache", False)) except Exception as e: EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), str(e))) self.feedback_q.put(taskname) finally: EdkLogger.debug(EdkLogger.DEBUG_9, "Worker %s: %s" % (os.getpid(), "Done")) self.feedback_q.put("Done") self.cache_q.put("CacheDone") def printStatus(self): print("Processs ID: %d Run %d modules in AutoGen " % (os.getpid(),len(AutoGen.Cache()))) print("Processs ID: %d Run %d modules in AutoGenInfo " % (os.getpid(),len(AutoGenInfo.GetCache()))) groupobj = {} for buildobj in BuildDB.BuildObject.GetCache().values(): if str(buildobj).lower().endswith("dec"): try: groupobj['dec'].append(str(buildobj)) except: groupobj['dec'] = [str(buildobj)] if str(buildobj).lower().endswith("dsc"): try: groupobj['dsc'].append(str(buildobj)) except: groupobj['dsc'] = [str(buildobj)] if str(buildobj).lower().endswith("inf"): try: groupobj['inf'].append(str(buildobj)) except: groupobj['inf'] = [str(buildobj)] print("Processs ID: %d Run %d pkg in WDB " % (os.getpid(),len(groupobj.get("dec",[])))) print("Processs ID: %d Run %d pla in WDB " % (os.getpid(),len(groupobj.get("dsc",[])))) print("Processs ID: %d Run %d inf in WDB " % (os.getpid(),len(groupobj.get("inf",[]))))