from __future__ import absolute_import import logger import os from urllib import parse as urlparse_local import config.base import socket import shutil # Fix parsing for nonstandard schemes urlparse_local.uses_netloc.extend(['bk', 'ssh', 'svn']) class Retriever(logger.Logger): def __init__(self, sourceControl, clArgs = None, argDB = None): logger.Logger.__init__(self, clArgs, argDB) self.sourceControl = sourceControl self.gitsubmodules = [] self.gitprereq = 1 self.git_urls = [] self.hg_urls = [] self.dir_urls = [] self.link_urls = [] self.tarball_urls = [] self.stamp = None self.ver = 'unknown' return def isGitURL(self, url): parsed = urlparse_local.urlparse(url) if (parsed[0] == 'git') or (parsed[0] == 'ssh' and parsed[2].endswith('.git')) or (parsed[0] == 'https' and parsed[2].endswith('.git')): return True elif os.path.isdir(url) and self.isDirectoryGitRepo(url): return True return False def setupURLs(self,packagename,urls,gitsubmodules,gitprereq): self.packagename = packagename self.gitsubmodules = gitsubmodules self.gitprereq = gitprereq for url in urls: parsed = urlparse_local.urlparse(url) if self.isGitURL(url): self.git_urls.append(self.removePrefix(url,'git://')) elif parsed[0] == 'hg'or (parsed[0] == 'ssh' and parsed[1].startswith('hg@')): self.hg_urls.append(self.removePrefix(url,'hg://')) elif parsed[0] == 'dir' or os.path.isdir(url): self.dir_urls.append(self.removePrefix(url,'dir://')) elif parsed[0] == 'link': self.link_urls.append(self.removePrefix(url,'link://')) else: self.tarball_urls.extend([url]) def isDirectoryGitRepo(self, directory): if not hasattr(self.sourceControl, 'git'): self.logPrint('git not found in self.sourceControl - cannot evaluate isDirectoryGitRepo(): '+directory) return False from config.base import Configure for loc in ['.git','']: cmd = '%s rev-parse --resolve-git-dir %s' % (self.sourceControl.git, os.path.join(directory,loc)) (output, error, ret) = Configure.executeShellCommand(cmd, checkCommand = Configure.passCheckCommand, log = self.log) if not ret: return True return False @staticmethod def removeTarget(t): if os.path.islink(t) or os.path.isfile(t): os.unlink(t) # same as os.remove(t) elif os.path.isdir(t): shutil.rmtree(t) @staticmethod def getDownloadFailureMessage(package, url, filename=None): slashFilename = '/'+filename if filename else '' return '''\ Unable to download package %s from: %s * If URL specified manually - perhaps there is a typo? * If your network is disconnected - please reconnect and rerun ./configure * Or perhaps you have a firewall blocking the download * You can run with --with-packages-download-dir=/adirectory and ./configure will instruct you what packages to download manually * or you can download the above URL manually, to /yourselectedlocation%s and use the configure option: --download-%s=/yourselectedlocation%s ''' % (package.upper(), url, slashFilename, package, slashFilename) @staticmethod def removePrefix(url,prefix): '''Replacement for str.removeprefix() supported only since Python 3.9''' if url.startswith(prefix): return url[len(prefix):] return url def generateURLs(self): if hasattr(self.sourceControl, 'git') and self.gitprereq: for url in self.git_urls: yield('git',url) else: self.logPrint('Git not found or gitprereq check failed! skipping giturls: '+str(self.git_urls)+'\n') if hasattr(self.sourceControl, 'hg'): for url in self.hg_urls: yield('hg',url) else: self.logPrint('Hg not found - skipping hgurls: '+str(self.hg_urls)+'\n') for url in self.dir_urls: yield('dir',url) for url in self.link_urls: yield('link',url) for url in self.tarball_urls: yield('tarball',url) def genericRetrieve(self,proto,url,root): '''Fetch package from version control repository or tarfile indicated by URL and extract it into root''' if proto == 'git': return self.gitRetrieve(url,root) elif proto == 'hg': return self.hgRetrieve(url,root) elif proto == 'dir': return self.dirRetrieve(url,root) elif proto == 'link': self.linkRetrieve(url,root) elif proto == 'tarball': self.tarballRetrieve(url,root) def dirRetrieve(self, url, root): self.logPrint('Retrieving %s as directory' % url, 3, 'install') if not os.path.isdir(url): raise RuntimeError('URL %s is not a directory' % url) t = os.path.join(root,os.path.basename(url)) self.removeTarget(t) shutil.copytree(url,t) def linkRetrieve(self, url, root): self.logPrint('Retrieving %s as link' % url, 3, 'install') if not os.path.isdir(url): raise RuntimeError('URL %s is not pointing to a directory' % url) t = os.path.join(root,os.path.basename(url)) self.removeTarget(t) os.symlink(os.path.abspath(url),t) def gitRetrieve(self, url, root): self.logPrint('Retrieving %s as git repo' % url, 3, 'install') if not hasattr(self.sourceControl, 'git'): raise RuntimeError('self.sourceControl.git not set') if os.path.isdir(url) and not self.isDirectoryGitRepo(url): raise RuntimeError('URL %s is a directory but not a git repository' % url) newgitrepo = os.path.join(root,'git.'+self.packagename) self.removeTarget(newgitrepo) try: submodopt ='' for itm in self.gitsubmodules: submodopt += ' --recurse-submodules='+itm config.base.Configure.executeShellCommand('%s clone %s %s %s' % (self.sourceControl.git, submodopt, url, newgitrepo), log = self.log, timeout = 120.0) except RuntimeError as e: self.logPrint('ERROR: '+str(e)) failureMessage = self.getDownloadFailureMessage(self.packagename, url) raise RuntimeError('Unable to clone '+self.packagename+'\n'+str(e)+'\n'+failureMessage) def hgRetrieve(self, url, root): self.logPrint('Retrieving %s as hg repo' % url, 3, 'install') if not hasattr(self.sourceControl, 'hg'): raise RuntimeError('self.sourceControl.hg not set') newgitrepo = os.path.join(root,'hg.'+self.packagename) self.removeTarget(newgitrepo) try: config.base.Configure.executeShellCommand('%s clone %s %s' % (self.sourceControl.hg, url, newgitrepo), log = self.log, timeout = 120.0) except RuntimeError as e: self.logPrint('ERROR: '+str(e)) failureMessage = self.getDownloadFailureMessage(self.packagename, url) raise RuntimeError('Unable to clone '+self.packagename+'\n'+str(e)+'\n'+failureMessage) def tarballRetrieve(self, url, root): parsed = urlparse_local.urlparse(url) filename = os.path.basename(parsed[2]) localFile = os.path.join(root,'_d_'+filename) self.logPrint('Retrieving %s as tarball to %s' % (url,localFile) , 3, 'install') ext = os.path.splitext(localFile)[1] if ext not in ['.bz2','.tbz','.gz','.tgz','.zip','.ZIP']: raise RuntimeError('Unknown compression type in URL: '+ url) self.removeTarget(localFile) if parsed[0] == 'file' and not parsed[1]: url = parsed[2] if os.path.exists(url): if not os.path.isfile(url): raise RuntimeError('Local path exists but is not a regular file: '+ url) # copy local file shutil.copyfile(url, localFile) else: # fetch remote file try: from urllib.request import Request, urlopen sav_timeout = socket.getdefaulttimeout() socket.setdefaulttimeout(30) req = Request(url) req.headers['User-Agent'] = 'PetscConfigure/'+self.ver with open(localFile, 'wb') as f: f.write(urlopen(req).read()) socket.setdefaulttimeout(sav_timeout) except Exception as e: socket.setdefaulttimeout(sav_timeout) failureMessage = self.getDownloadFailureMessage(self.packagename, url, filename) raise RuntimeError(str(e)+'\n'+failureMessage) self.logPrint('Extracting '+localFile) if ext in ['.zip','.ZIP']: config.base.Configure.executeShellCommand('cd '+root+'; unzip '+localFile, log = self.log) output = config.base.Configure.executeShellCommand('cd '+root+'; zipinfo -1 '+localFile+' | head -n 1', log = self.log) dirname = os.path.normpath(output[0].strip()) else: failureMessage = '''\ Downloaded package %s from: %s is not a tarball. [or installed python cannot process compressed files] * If you are behind a firewall - please fix your proxy and rerun ./configure For example at LANL you may need to set the environmental variable http_proxy (or HTTP_PROXY?) to http://proxyout.lanl.gov * You can run with --with-packages-download-dir=/adirectory and ./configure will instruct you what packages to download manually * or you can download the above URL manually, to /yourselectedlocation/%s and use the configure option: --download-%s=/yourselectedlocation/%s ''' % (self.packagename.upper(), url, filename, self.packagename, filename) import tarfile try: tf = tarfile.open(os.path.join(root, localFile)) except tarfile.ReadError as e: raise RuntimeError(str(e)+'\n'+failureMessage) if not tf: raise RuntimeError(failureMessage) #git puts 'pax_global_header' as the first entry and some tar utils process this as a file firstname = tf.getnames()[0] if firstname == 'pax_global_header': firstmember = tf.getmembers()[1] else: firstmember = tf.getmembers()[0] # some tarfiles list packagename/ but some list packagename/filename in the first entry if firstmember.isdir(): dirname = firstmember.name else: dirname = os.path.dirname(firstmember.name) tf.extractall(root) tf.close() # fix file permissions for the untared tarballs. try: # check if 'dirname' is set' if dirname: config.base.Configure.executeShellCommand('cd '+root+'; chmod -R a+r '+dirname+';find '+dirname + r' -type d -name "*" -exec chmod a+rx {} \;', log = self.log) else: self.logPrintBox('WARNING: Could not determine dirname extracted by '+localFile+' to fix file permissions') except RuntimeError as e: raise RuntimeError('Error changing permissions for '+dirname+' obtained from '+localFile+ ' : '+str(e)) os.unlink(localFile)