diff options
Diffstat (limited to 'tools/buildman/toolchain.py')
| -rw-r--r-- | tools/buildman/toolchain.py | 270 |
1 files changed, 250 insertions, 20 deletions
diff --git a/tools/buildman/toolchain.py b/tools/buildman/toolchain.py index 27dc318..d4c5d4a 100644 --- a/tools/buildman/toolchain.py +++ b/tools/buildman/toolchain.py @@ -5,11 +5,42 @@ import re import glob +from HTMLParser import HTMLParser import os +import sys +import tempfile +import urllib2 import bsettings import command +# Simple class to collect links from a page +class MyHTMLParser(HTMLParser): + def __init__(self, arch): + """Create a new parser + + After the parser runs, self.links will be set to a list of the links + to .xz archives found in the page, and self.arch_link will be set to + the one for the given architecture (or None if not found). + + Args: + arch: Architecture to search for + """ + HTMLParser.__init__(self) + self.arch_link = None + self.links = [] + self._match = '_%s-' % arch + + def handle_starttag(self, tag, attrs): + if tag == 'a': + for tag, value in attrs: + if tag == 'href': + if value and value.endswith('.xz'): + self.links.append(value) + if self._match in value: + self.arch_link = value + + class Toolchain: """A single toolchain @@ -20,7 +51,6 @@ class Toolchain: arch: Architecture of toolchain as determined from the first component of the filename. E.g. arm-linux-gcc becomes arm """ - def __init__(self, fname, test, verbose=False): """Create a new toolchain object. @@ -30,11 +60,18 @@ class Toolchain: """ self.gcc = fname self.path = os.path.dirname(fname) - self.cross = os.path.basename(fname)[:-3] + + # Find the CROSS_COMPILE prefix to use for U-Boot. For example, + # 'arm-linux-gnueabihf-gcc' turns into 'arm-linux-gnueabihf-'. + basename = os.path.basename(fname) + pos = basename.rfind('-') + self.cross = basename[:pos + 1] if pos != -1 else '' + + # The architecture is the first part of the name pos = self.cross.find('-') self.arch = self.cross[:pos] if pos != -1 else 'sandbox' - env = self.MakeEnvironment() + env = self.MakeEnvironment(False) # As a basic sanity check, run the C compiler with --version cmd = [fname, '--version'] @@ -74,15 +111,23 @@ class Toolchain: return prio return prio - def MakeEnvironment(self): + def MakeEnvironment(self, full_path): """Returns an environment for using the toolchain. - Thie takes the current environment, adds CROSS_COMPILE and - augments PATH so that the toolchain will operate correctly. + Thie takes the current environment and adds CROSS_COMPILE so that + the tool chain will operate correctly. + + Args: + full_path: Return the full path in CROSS_COMPILE and don't set + PATH """ env = dict(os.environ) - env['CROSS_COMPILE'] = self.cross - env['PATH'] += (':' + self.path) + if full_path: + env['CROSS_COMPILE'] = os.path.join(self.path, self.cross) + else: + env['CROSS_COMPILE'] = self.cross + env['PATH'] = self.path + ':' + env['PATH'] + return env @@ -101,18 +146,29 @@ class Toolchains: self.paths = [] self._make_flags = dict(bsettings.GetItems('make-flags')) - def GetSettings(self): + def GetPathList(self): + """Get a list of available toolchain paths + + Returns: + List of strings, each a path to a toolchain mentioned in the + [toolchain] section of the settings file. + """ toolchains = bsettings.GetItems('toolchain') if not toolchains: print ("Warning: No tool chains - please add a [toolchain] section" " to your buildman config file %s. See README for details" % bsettings.config_fname) + paths = [] for name, value in toolchains: if '*' in value: - self.paths += glob.glob(value) + paths += glob.glob(value) else: - self.paths.append(value) + paths.append(value) + return paths + + def GetSettings(self): + self.paths += self.GetPathList() def Add(self, fname, test=True, verbose=False): """Add a toolchain to our list @@ -132,6 +188,24 @@ class Toolchains: if add_it: self.toolchains[toolchain.arch] = toolchain + def ScanPath(self, path, verbose): + """Scan a path for a valid toolchain + + Args: + path: Path to scan + verbose: True to print out progress information + Returns: + Filename of C compiler if found, else None + """ + for subdir in ['.', 'bin', 'usr/bin']: + dirname = os.path.join(path, subdir) + if verbose: print " - looking in '%s'" % dirname + for fname in glob.glob(dirname + '/*gcc'): + if verbose: print " - found '%s'" % fname + return fname + return None + + def Scan(self, verbose): """Scan for available toolchains and select the best for each arch. @@ -145,12 +219,9 @@ class Toolchains: if verbose: print 'Scanning for tool chains' for path in self.paths: if verbose: print " - scanning path '%s'" % path - for subdir in ['.', 'bin', 'usr/bin']: - dirname = os.path.join(path, subdir) - if verbose: print " - looking in '%s'" % dirname - for fname in glob.glob(dirname + '/*gcc'): - if verbose: print " - found '%s'" % fname - self.Add(fname, True, verbose) + fname = self.ScanPath(path, verbose) + if fname: + self.Add(fname, True, verbose) def List(self): """List out the selected toolchains for each architecture""" @@ -170,9 +241,11 @@ class Toolchains: returns: toolchain object, or None if none found """ - for name, value in bsettings.GetItems('toolchain-alias'): - if arch == name: - arch = value + for tag, value in bsettings.GetItems('toolchain-alias'): + if arch == tag: + for alias in value.split(): + if alias in self.toolchains: + return self.toolchains[alias] if not arch in self.toolchains: raise ValueError, ("No tool chain found for arch '%s'" % arch) @@ -247,3 +320,160 @@ class Toolchains: else: i += 1 return args + + def LocateArchUrl(self, fetch_arch): + """Find a toolchain available online + + Look in standard places for available toolchains. At present the + only standard place is at kernel.org. + + Args: + arch: Architecture to look for, or 'list' for all + Returns: + If fetch_arch is 'list', a tuple: + Machine architecture (e.g. x86_64) + List of toolchains + else + URL containing this toolchain, if avaialble, else None + """ + arch = command.OutputOneLine('uname', '-m') + base = 'https://www.kernel.org/pub/tools/crosstool/files/bin' + versions = ['4.6.3', '4.6.2', '4.5.1', '4.2.4'] + links = [] + for version in versions: + url = '%s/%s/%s/' % (base, arch, version) + print 'Checking: %s' % url + response = urllib2.urlopen(url) + html = response.read() + parser = MyHTMLParser(fetch_arch) + parser.feed(html) + if fetch_arch == 'list': + links += parser.links + elif parser.arch_link: + return url + parser.arch_link + if fetch_arch == 'list': + return arch, links + return None + + def Download(self, url): + """Download a file to a temporary directory + + Args: + url: URL to download + Returns: + Tuple: + Temporary directory name + Full path to the downloaded archive file in that directory, + or None if there was an error while downloading + """ + print "Downloading: %s" % url + leaf = url.split('/')[-1] + tmpdir = tempfile.mkdtemp('.buildman') + response = urllib2.urlopen(url) + fname = os.path.join(tmpdir, leaf) + fd = open(fname, 'wb') + meta = response.info() + size = int(meta.getheaders("Content-Length")[0]) + done = 0 + block_size = 1 << 16 + status = '' + + # Read the file in chunks and show progress as we go + while True: + buffer = response.read(block_size) + if not buffer: + print chr(8) * (len(status) + 1), '\r', + break + + done += len(buffer) + fd.write(buffer) + status = r"%10d MiB [%3d%%]" % (done / 1024 / 1024, + done * 100 / size) + status = status + chr(8) * (len(status) + 1) + print status, + sys.stdout.flush() + fd.close() + if done != size: + print 'Error, failed to download' + os.remove(fname) + fname = None + return tmpdir, fname + + def Unpack(self, fname, dest): + """Unpack a tar file + + Args: + fname: Filename to unpack + dest: Destination directory + Returns: + Directory name of the first entry in the archive, without the + trailing / + """ + stdout = command.Output('tar', 'xvfJ', fname, '-C', dest) + return stdout.splitlines()[0][:-1] + + def TestSettingsHasPath(self, path): + """Check if builmand will find this toolchain + + Returns: + True if the path is in settings, False if not + """ + paths = self.GetPathList() + return path in paths + + def ListArchs(self): + """List architectures with available toolchains to download""" + host_arch, archives = self.LocateArchUrl('list') + re_arch = re.compile('[-a-z0-9.]*_([^-]*)-.*') + arch_set = set() + for archive in archives: + # Remove the host architecture from the start + arch = re_arch.match(archive[len(host_arch):]) + if arch: + arch_set.add(arch.group(1)) + return sorted(arch_set) + + def FetchAndInstall(self, arch): + """Fetch and install a new toolchain + + arch: + Architecture to fetch, or 'list' to list + """ + # Fist get the URL for this architecture + url = self.LocateArchUrl(arch) + if not url: + print ("Cannot find toolchain for arch '%s' - use 'list' to list" % + arch) + return 2 + home = os.environ['HOME'] + dest = os.path.join(home, '.buildman-toolchains') + if not os.path.exists(dest): + os.mkdir(dest) + + # Download the tar file for this toolchain and unpack it + tmpdir, tarfile = self.Download(url) + if not tarfile: + return 1 + print 'Unpacking to: %s' % dest, + sys.stdout.flush() + path = self.Unpack(tarfile, dest) + os.remove(tarfile) + os.rmdir(tmpdir) + print + + # Check that the toolchain works + print 'Testing' + dirpath = os.path.join(dest, path) + compiler_fname = self.ScanPath(dirpath, True) + if not compiler_fname: + print 'Could not locate C compiler - fetch failed.' + return 1 + toolchain = Toolchain(compiler_fname, True, True) + + # Make sure that it will be found by buildman + if not self.TestSettingsHasPath(dirpath): + print ("Adding 'download' to config file '%s'" % + bsettings.config_fname) + tools_dir = os.path.dirname(dirpath) + bsettings.SetItem('toolchain', 'download', '%s/*' % tools_dir) + return 0 |