+++ /dev/null
-From c05aa0364280803d8274e260a739553d588ea052 Mon Sep 17 00:00:00 2001
-From: Simon Glass <sjg@chromium.org>
-Date: Thu, 31 Oct 2019 07:42:53 -0600
-Subject: [PATCH] buildman: Convert to Python 3
-
-Convert buildman to Python 3 and make it use that, to meet the 2020
-deadline.
-
-Upstream-Status: Backport
-
-Signed-off-by: Simon Glass <sjg@chromium.org>
----
- tools/buildman/board.py | 9 +--
- tools/buildman/bsettings.py | 20 +++----
- tools/buildman/builder.py | 47 ++++++++--------
- tools/buildman/builderthread.py | 24 ++++----
- tools/buildman/buildman.py | 10 ++--
- tools/buildman/control.py | 44 +++++++--------
- tools/buildman/func_test.py | 16 +++---
- tools/buildman/test.py | 22 ++++----
- tools/buildman/toolchain.py | 99 +++++++++++++++++----------------
- 9 files changed, 146 insertions(+), 145 deletions(-)
-
-diff --git a/tools/buildman/board.py b/tools/buildman/board.py
-index 2a1d021574..447aaabea8 100644
---- a/tools/buildman/board.py
-+++ b/tools/buildman/board.py
-@@ -1,6 +1,7 @@
- # SPDX-License-Identifier: GPL-2.0+
- # Copyright (c) 2012 The Chromium OS Authors.
-
-+from collections import OrderedDict
- import re
-
- class Expr:
-@@ -120,7 +121,7 @@ class Boards:
- Args:
- fname: Filename of boards.cfg file
- """
-- with open(fname, 'r') as fd:
-+ with open(fname, 'r', encoding='utf-8') as fd:
- for line in fd:
- if line[0] == '#':
- continue
-@@ -155,7 +156,7 @@ class Boards:
- key is board.target
- value is board
- """
-- board_dict = {}
-+ board_dict = OrderedDict()
- for board in self._boards:
- board_dict[board.target] = board
- return board_dict
-@@ -166,7 +167,7 @@ class Boards:
- Returns:
- List of Board objects that are marked selected
- """
-- board_dict = {}
-+ board_dict = OrderedDict()
- for board in self._boards:
- if board.build_it:
- board_dict[board.target] = board
-@@ -259,7 +260,7 @@ class Boards:
- due to each argument, arranged by argument.
- List of errors found
- """
-- result = {}
-+ result = OrderedDict()
- warnings = []
- terms = self._BuildTerms(args)
-
-diff --git a/tools/buildman/bsettings.py b/tools/buildman/bsettings.py
-index 03d7439aa5..0b7208da37 100644
---- a/tools/buildman/bsettings.py
-+++ b/tools/buildman/bsettings.py
-@@ -1,9 +1,9 @@
- # SPDX-License-Identifier: GPL-2.0+
- # Copyright (c) 2012 The Chromium OS Authors.
-
--import ConfigParser
-+import configparser
- import os
--import StringIO
-+import io
-
-
- def Setup(fname=''):
-@@ -15,20 +15,20 @@ def Setup(fname=''):
- global settings
- global config_fname
-
-- settings = ConfigParser.SafeConfigParser()
-+ settings = configparser.SafeConfigParser()
- if fname is not None:
- config_fname = fname
- if config_fname == '':
- config_fname = '%s/.buildman' % os.getenv('HOME')
- if not os.path.exists(config_fname):
-- print 'No config file found ~/.buildman\nCreating one...\n'
-+ print('No config file found ~/.buildman\nCreating one...\n')
- CreateBuildmanConfigFile(config_fname)
-- print 'To install tool chains, please use the --fetch-arch option'
-+ print('To install tool chains, please use the --fetch-arch option')
- if config_fname:
- settings.read(config_fname)
-
- def AddFile(data):
-- settings.readfp(StringIO.StringIO(data))
-+ settings.readfp(io.StringIO(data))
-
- def GetItems(section):
- """Get the items from a section of the config.
-@@ -41,7 +41,7 @@ def GetItems(section):
- """
- try:
- return settings.items(section)
-- except ConfigParser.NoSectionError as e:
-+ except configparser.NoSectionError as e:
- return []
- except:
- raise
-@@ -68,10 +68,10 @@ def CreateBuildmanConfigFile(config_fname):
- try:
- f = open(config_fname, 'w')
- except IOError:
-- print "Couldn't create buildman config file '%s'\n" % config_fname
-+ print("Couldn't create buildman config file '%s'\n" % config_fname)
- raise
-
-- print >>f, '''[toolchain]
-+ print('''[toolchain]
- # name = path
- # e.g. x86 = /opt/gcc-4.6.3-nolibc/x86_64-linux
-
-@@ -93,5 +93,5 @@ openrisc = or1k
- # snapper-boards=ENABLE_AT91_TEST=1
- # snapper9260=${snapper-boards} BUILD_TAG=442
- # snapper9g45=${snapper-boards} BUILD_TAG=443
--'''
-+''', file=f)
- f.close();
-diff --git a/tools/buildman/builder.py b/tools/buildman/builder.py
-index fbb236676c..cfbe4c26b1 100644
---- a/tools/buildman/builder.py
-+++ b/tools/buildman/builder.py
-@@ -9,7 +9,7 @@ from datetime import datetime, timedelta
- import glob
- import os
- import re
--import Queue
-+import queue
- import shutil
- import signal
- import string
-@@ -92,11 +92,10 @@ u-boot/ source directory
- """
-
- # Possible build outcomes
--OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4)
-+OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = list(range(4))
-
- # Translate a commit subject into a valid filename (and handle unicode)
--trans_valid_chars = string.maketrans('/: ', '---')
--trans_valid_chars = trans_valid_chars.decode('latin-1')
-+trans_valid_chars = str.maketrans('/: ', '---')
-
- BASE_CONFIG_FILENAMES = [
- 'u-boot.cfg', 'u-boot-spl.cfg', 'u-boot-tpl.cfg'
-@@ -122,8 +121,8 @@ class Config:
- def __hash__(self):
- val = 0
- for fname in self.config:
-- for key, value in self.config[fname].iteritems():
-- print key, value
-+ for key, value in self.config[fname].items():
-+ print(key, value)
- val = val ^ hash(key) & hash(value)
- return val
-
-@@ -293,8 +292,8 @@ class Builder:
- self._re_dtb_warning = re.compile('(.*): Warning .*')
- self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*')
-
-- self.queue = Queue.Queue()
-- self.out_queue = Queue.Queue()
-+ self.queue = queue.Queue()
-+ self.out_queue = queue.Queue()
- for i in range(self.num_threads):
- t = builderthread.BuilderThread(self, i, incremental,
- per_board_out_dir)
-@@ -781,7 +780,7 @@ class Builder:
- config = {}
- environment = {}
-
-- for board in boards_selected.itervalues():
-+ for board in boards_selected.values():
- outcome = self.GetBuildOutcome(commit_upto, board.target,
- read_func_sizes, read_config,
- read_environment)
-@@ -814,13 +813,13 @@ class Builder:
- tconfig = Config(self.config_filenames, board.target)
- for fname in self.config_filenames:
- if outcome.config:
-- for key, value in outcome.config[fname].iteritems():
-+ for key, value in outcome.config[fname].items():
- tconfig.Add(fname, key, value)
- config[board.target] = tconfig
-
- tenvironment = Environment(board.target)
- if outcome.environment:
-- for key, value in outcome.environment.iteritems():
-+ for key, value in outcome.environment.items():
- tenvironment.Add(key, value)
- environment[board.target] = tenvironment
-
-@@ -1040,12 +1039,12 @@ class Builder:
-
- # We now have a list of image size changes sorted by arch
- # Print out a summary of these
-- for arch, target_list in arch_list.iteritems():
-+ for arch, target_list in arch_list.items():
- # Get total difference for each type
- totals = {}
- for result in target_list:
- total = 0
-- for name, diff in result.iteritems():
-+ for name, diff in result.items():
- if name.startswith('_'):
- continue
- total += diff
-@@ -1250,7 +1249,7 @@ class Builder:
- if self._show_unknown:
- self.AddOutcome(board_selected, arch_list, unknown_boards, '?',
- self.col.MAGENTA)
-- for arch, target_list in arch_list.iteritems():
-+ for arch, target_list in arch_list.items():
- Print('%10s: %s' % (arch, target_list))
- self._error_lines += 1
- if better_err:
-@@ -1283,13 +1282,13 @@ class Builder:
- environment_minus = {}
- environment_change = {}
- base = tbase.environment
-- for key, value in tenvironment.environment.iteritems():
-+ for key, value in tenvironment.environment.items():
- if key not in base:
- environment_plus[key] = value
-- for key, value in base.iteritems():
-+ for key, value in base.items():
- if key not in tenvironment.environment:
- environment_minus[key] = value
-- for key, value in base.iteritems():
-+ for key, value in base.items():
- new_value = tenvironment.environment.get(key)
- if new_value and value != new_value:
- desc = '%s -> %s' % (value, new_value)
-@@ -1342,15 +1341,15 @@ class Builder:
- config_minus = {}
- config_change = {}
- base = tbase.config[name]
-- for key, value in tconfig.config[name].iteritems():
-+ for key, value in tconfig.config[name].items():
- if key not in base:
- config_plus[key] = value
- all_config_plus[key] = value
-- for key, value in base.iteritems():
-+ for key, value in base.items():
- if key not in tconfig.config[name]:
- config_minus[key] = value
- all_config_minus[key] = value
-- for key, value in base.iteritems():
-+ for key, value in base.items():
- new_value = tconfig.config.get(key)
- if new_value and value != new_value:
- desc = '%s -> %s' % (value, new_value)
-@@ -1368,7 +1367,7 @@ class Builder:
- summary[target] = '\n'.join(lines)
-
- lines_by_target = {}
-- for target, lines in summary.iteritems():
-+ for target, lines in summary.items():
- if lines in lines_by_target:
- lines_by_target[lines].append(target)
- else:
-@@ -1392,7 +1391,7 @@ class Builder:
- Print('%s:' % arch)
- _OutputConfigInfo(lines)
-
-- for lines, targets in lines_by_target.iteritems():
-+ for lines, targets in lines_by_target.items():
- if not lines:
- continue
- Print('%s :' % ' '.join(sorted(targets)))
-@@ -1463,7 +1462,7 @@ class Builder:
- commits: Selected commits to build
- """
- # First work out how many commits we will build
-- count = (self.commit_count + self._step - 1) / self._step
-+ count = (self.commit_count + self._step - 1) // self._step
- self.count = len(board_selected) * count
- self.upto = self.warned = self.fail = 0
- self._timestamps = collections.deque()
-@@ -1566,7 +1565,7 @@ class Builder:
- self.ProcessResult(None)
-
- # Create jobs to build all commits for each board
-- for brd in board_selected.itervalues():
-+ for brd in board_selected.values():
- job = builderthread.BuilderJob()
- job.board = brd
- job.commits = commits
-diff --git a/tools/buildman/builderthread.py b/tools/buildman/builderthread.py
-index 8a9d47cd5e..570c1f6595 100644
---- a/tools/buildman/builderthread.py
-+++ b/tools/buildman/builderthread.py
-@@ -28,7 +28,7 @@ def Mkdir(dirname, parents = False):
- except OSError as err:
- if err.errno == errno.EEXIST:
- if os.path.realpath('.') == os.path.realpath(dirname):
-- print "Cannot create the current working directory '%s'!" % dirname
-+ print("Cannot create the current working directory '%s'!" % dirname)
- sys.exit(1)
- pass
- else:
-@@ -291,15 +291,13 @@ class BuilderThread(threading.Thread):
- outfile = os.path.join(build_dir, 'log')
- with open(outfile, 'w') as fd:
- if result.stdout:
-- # We don't want unicode characters in log files
-- fd.write(result.stdout.decode('UTF-8').encode('ASCII', 'replace'))
-+ fd.write(result.stdout)
-
- errfile = self.builder.GetErrFile(result.commit_upto,
- result.brd.target)
- if result.stderr:
- with open(errfile, 'w') as fd:
-- # We don't want unicode characters in log files
-- fd.write(result.stderr.decode('UTF-8').encode('ASCII', 'replace'))
-+ fd.write(result.stderr)
- elif os.path.exists(errfile):
- os.remove(errfile)
-
-@@ -314,17 +312,17 @@ class BuilderThread(threading.Thread):
- else:
- fd.write('%s' % result.return_code)
- with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
-- print >>fd, 'gcc', result.toolchain.gcc
-- print >>fd, 'path', result.toolchain.path
-- print >>fd, 'cross', result.toolchain.cross
-- print >>fd, 'arch', result.toolchain.arch
-+ print('gcc', result.toolchain.gcc, file=fd)
-+ print('path', result.toolchain.path, file=fd)
-+ print('cross', result.toolchain.cross, file=fd)
-+ print('arch', result.toolchain.arch, file=fd)
- fd.write('%s' % result.return_code)
-
- # Write out the image and function size information and an objdump
- env = result.toolchain.MakeEnvironment(self.builder.full_path)
- with open(os.path.join(build_dir, 'env'), 'w') as fd:
- for var in sorted(env.keys()):
-- print >>fd, '%s="%s"' % (var, env[var])
-+ print('%s="%s"' % (var, env[var]), file=fd)
- lines = []
- for fname in ['u-boot', 'spl/u-boot-spl']:
- cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname]
-@@ -335,7 +333,7 @@ class BuilderThread(threading.Thread):
- nm = self.builder.GetFuncSizesFile(result.commit_upto,
- result.brd.target, fname)
- with open(nm, 'w') as fd:
-- print >>fd, nm_result.stdout,
-+ print(nm_result.stdout, end=' ', file=fd)
-
- cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname]
- dump_result = command.RunPipe([cmd], capture=True,
-@@ -346,7 +344,7 @@ class BuilderThread(threading.Thread):
- objdump = self.builder.GetObjdumpFile(result.commit_upto,
- result.brd.target, fname)
- with open(objdump, 'w') as fd:
-- print >>fd, dump_result.stdout,
-+ print(dump_result.stdout, end=' ', file=fd)
- for line in dump_result.stdout.splitlines():
- fields = line.split()
- if len(fields) > 5 and fields[1] == '.rodata':
-@@ -378,7 +376,7 @@ class BuilderThread(threading.Thread):
- sizes = self.builder.GetSizesFile(result.commit_upto,
- result.brd.target)
- with open(sizes, 'w') as fd:
-- print >>fd, '\n'.join(lines)
-+ print('\n'.join(lines), file=fd)
-
- # Write out the configuration files, with a special case for SPL
- for dirname in ['', 'spl', 'tpl']:
-diff --git a/tools/buildman/buildman.py b/tools/buildman/buildman.py
-index f17aa15e7c..30a8690f93 100755
---- a/tools/buildman/buildman.py
-+++ b/tools/buildman/buildman.py
-@@ -1,4 +1,4 @@
--#!/usr/bin/env python2
-+#!/usr/bin/env python3
- # SPDX-License-Identifier: GPL-2.0+
- #
- # Copyright (c) 2012 The Chromium OS Authors.
-@@ -6,6 +6,8 @@
-
- """See README for more information"""
-
-+from __future__ import print_function
-+
- import multiprocessing
- import os
- import re
-@@ -46,11 +48,11 @@ def RunTests(skip_net_tests):
- suite = unittest.TestLoader().loadTestsFromTestCase(module)
- suite.run(result)
-
-- print result
-+ print(result)
- for test, err in result.errors:
-- print err
-+ print(err)
- for test, err in result.failures:
-- print err
-+ print(err)
-
-
- options, args = cmdline.ParseArgs()
-diff --git a/tools/buildman/control.py b/tools/buildman/control.py
-index 9787b86747..216012d001 100644
---- a/tools/buildman/control.py
-+++ b/tools/buildman/control.py
-@@ -30,7 +30,7 @@ def GetActionSummary(is_summary, commits, selected, options):
- """
- if commits:
- count = len(commits)
-- count = (count + options.step - 1) / options.step
-+ count = (count + options.step - 1) // options.step
- commit_str = '%d commit%s' % (count, GetPlural(count))
- else:
- commit_str = 'current source'
-@@ -59,31 +59,31 @@ def ShowActions(series, why_selected, boards_selected, builder, options,
- board_warnings: List of warnings obtained from board selected
- """
- col = terminal.Color()
-- print 'Dry run, so not doing much. But I would do this:'
-- print
-+ print('Dry run, so not doing much. But I would do this:')
-+ print()
- if series:
- commits = series.commits
- else:
- commits = None
-- print GetActionSummary(False, commits, boards_selected,
-- options)
-- print 'Build directory: %s' % builder.base_dir
-+ print(GetActionSummary(False, commits, boards_selected,
-+ options))
-+ print('Build directory: %s' % builder.base_dir)
- if commits:
- for upto in range(0, len(series.commits), options.step):
- commit = series.commits[upto]
-- print ' ', col.Color(col.YELLOW, commit.hash[:8], bright=False),
-- print commit.subject
-- print
-+ print(' ', col.Color(col.YELLOW, commit.hash[:8], bright=False), end=' ')
-+ print(commit.subject)
-+ print()
- for arg in why_selected:
- if arg != 'all':
-- print arg, ': %d boards' % len(why_selected[arg])
-+ print(arg, ': %d boards' % len(why_selected[arg]))
- if options.verbose:
-- print ' %s' % ' '.join(why_selected[arg])
-- print ('Total boards to build for each commit: %d\n' %
-- len(why_selected['all']))
-+ print(' %s' % ' '.join(why_selected[arg]))
-+ print(('Total boards to build for each commit: %d\n' %
-+ len(why_selected['all'])))
- if board_warnings:
- for warning in board_warnings:
-- print col.Color(col.YELLOW, warning)
-+ print(col.Color(col.YELLOW, warning))
-
- def CheckOutputDir(output_dir):
- """Make sure that the output directory is not within the current directory
-@@ -146,17 +146,17 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
- if options.fetch_arch:
- if options.fetch_arch == 'list':
- sorted_list = toolchains.ListArchs()
-- print col.Color(col.BLUE, 'Available architectures: %s\n' %
-- ' '.join(sorted_list))
-+ print(col.Color(col.BLUE, 'Available architectures: %s\n' %
-+ ' '.join(sorted_list)))
- return 0
- else:
- fetch_arch = options.fetch_arch
- if fetch_arch == 'all':
- fetch_arch = ','.join(toolchains.ListArchs())
-- print col.Color(col.CYAN, '\nDownloading toolchains: %s' %
-- fetch_arch)
-+ print(col.Color(col.CYAN, '\nDownloading toolchains: %s' %
-+ fetch_arch))
- for arch in fetch_arch.split(','):
-- print
-+ print()
- ret = toolchains.FetchAndInstall(arch)
- if ret:
- return ret
-@@ -167,7 +167,7 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
- toolchains.Scan(options.list_tool_chains and options.verbose)
- if options.list_tool_chains:
- toolchains.List()
-- print
-+ print()
- return 0
-
- # Work out how many commits to build. We want to build everything on the
-@@ -191,7 +191,7 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
- sys.exit(col.Color(col.RED, "Range '%s' has no commits" %
- options.branch))
- if msg:
-- print col.Color(col.YELLOW, msg)
-+ print(col.Color(col.YELLOW, msg))
- count += 1 # Build upstream commit also
-
- if not count:
-@@ -268,7 +268,7 @@ def DoBuildman(options, args, toolchains=None, make_func=None, boards=None,
- options.threads = min(multiprocessing.cpu_count(), len(selected))
- if not options.jobs:
- options.jobs = max(1, (multiprocessing.cpu_count() +
-- len(selected) - 1) / len(selected))
-+ len(selected) - 1) // len(selected))
-
- if not options.step:
- options.step = len(series.commits) - 1
-diff --git a/tools/buildman/func_test.py b/tools/buildman/func_test.py
-index f90b8ea7f5..4c3d497294 100644
---- a/tools/buildman/func_test.py
-+++ b/tools/buildman/func_test.py
-@@ -270,7 +270,7 @@ class TestFunctional(unittest.TestCase):
- stdout=''.join(commit_log[:count]))
-
- # Not handled, so abort
-- print 'git log', args
-+ print('git log', args)
- sys.exit(1)
-
- def _HandleCommandGitConfig(self, args):
-@@ -286,7 +286,7 @@ class TestFunctional(unittest.TestCase):
- stdout='refs/heads/master\n')
-
- # Not handled, so abort
-- print 'git config', args
-+ print('git config', args)
- sys.exit(1)
-
- def _HandleCommandGit(self, in_args):
-@@ -320,7 +320,7 @@ class TestFunctional(unittest.TestCase):
- return command.CommandResult(return_code=0)
-
- # Not handled, so abort
-- print 'git', git_args, sub_cmd, args
-+ print('git', git_args, sub_cmd, args)
- sys.exit(1)
-
- def _HandleCommandNm(self, args):
-@@ -351,7 +351,7 @@ class TestFunctional(unittest.TestCase):
- if pipe_list[1] == ['wc', '-l']:
- wc = True
- else:
-- print 'invalid pipe', kwargs
-+ print('invalid pipe', kwargs)
- sys.exit(1)
- cmd = pipe_list[0][0]
- args = pipe_list[0][1:]
-@@ -371,7 +371,7 @@ class TestFunctional(unittest.TestCase):
-
- if not result:
- # Not handled, so abort
-- print 'unknown command', kwargs
-+ print('unknown command', kwargs)
- sys.exit(1)
-
- if wc:
-@@ -404,14 +404,14 @@ class TestFunctional(unittest.TestCase):
- return command.CommandResult(return_code=0)
-
- # Not handled, so abort
-- print 'make', stage
-+ print('make', stage)
- sys.exit(1)
-
- # Example function to print output lines
- def print_lines(self, lines):
-- print len(lines)
-+ print(len(lines))
- for line in lines:
-- print line
-+ print(line)
- #self.print_lines(terminal.GetPrintTestLines())
-
- def testNoBoards(self):
-diff --git a/tools/buildman/test.py b/tools/buildman/test.py
-index ed99b9375c..b4e28d6867 100644
---- a/tools/buildman/test.py
-+++ b/tools/buildman/test.py
-@@ -212,11 +212,11 @@ class TestBuild(unittest.TestCase):
- self.assertEqual(lines[1].text, '02: %s' % commits[1][1])
-
- col = terminal.Color()
-- self.assertSummary(lines[2].text, 'sandbox', 'w+', ['board4'],
-+ self.assertSummary(lines[2].text, 'arm', 'w+', ['board1'],
- outcome=OUTCOME_WARN)
-- self.assertSummary(lines[3].text, 'arm', 'w+', ['board1'],
-+ self.assertSummary(lines[3].text, 'powerpc', 'w+', ['board2', 'board3'],
- outcome=OUTCOME_WARN)
-- self.assertSummary(lines[4].text, 'powerpc', 'w+', ['board2', 'board3'],
-+ self.assertSummary(lines[4].text, 'sandbox', 'w+', ['board4'],
- outcome=OUTCOME_WARN)
-
- # Second commit: The warnings should be listed
-@@ -226,10 +226,10 @@ class TestBuild(unittest.TestCase):
-
- # Third commit: Still fails
- self.assertEqual(lines[6].text, '03: %s' % commits[2][1])
-- self.assertSummary(lines[7].text, 'sandbox', '+', ['board4'])
-- self.assertSummary(lines[8].text, 'arm', '', ['board1'],
-+ self.assertSummary(lines[7].text, 'arm', '', ['board1'],
- outcome=OUTCOME_OK)
-- self.assertSummary(lines[9].text, 'powerpc', '+', ['board2', 'board3'])
-+ self.assertSummary(lines[8].text, 'powerpc', '+', ['board2', 'board3'])
-+ self.assertSummary(lines[9].text, 'sandbox', '+', ['board4'])
-
- # Expect a compiler error
- self.assertEqual(lines[10].text, '+%s' %
-@@ -237,8 +237,6 @@ class TestBuild(unittest.TestCase):
-
- # Fourth commit: Compile errors are fixed, just have warning for board3
- self.assertEqual(lines[11].text, '04: %s' % commits[3][1])
-- self.assertSummary(lines[12].text, 'sandbox', 'w+', ['board4'],
-- outcome=OUTCOME_WARN)
- expect = '%10s: ' % 'powerpc'
- expect += ' ' + col.Color(col.GREEN, '')
- expect += ' '
-@@ -246,7 +244,9 @@ class TestBuild(unittest.TestCase):
- expect += ' ' + col.Color(col.YELLOW, 'w+')
- expect += ' '
- expect += col.Color(col.YELLOW, ' %s' % 'board3')
-- self.assertEqual(lines[13].text, expect)
-+ self.assertEqual(lines[12].text, expect)
-+ self.assertSummary(lines[13].text, 'sandbox', 'w+', ['board4'],
-+ outcome=OUTCOME_WARN)
-
- # Compile error fixed
- self.assertEqual(lines[14].text, '-%s' %
-@@ -259,9 +259,9 @@ class TestBuild(unittest.TestCase):
-
- # Fifth commit
- self.assertEqual(lines[16].text, '05: %s' % commits[4][1])
-- self.assertSummary(lines[17].text, 'sandbox', '+', ['board4'])
-- self.assertSummary(lines[18].text, 'powerpc', '', ['board3'],
-+ self.assertSummary(lines[17].text, 'powerpc', '', ['board3'],
- outcome=OUTCOME_OK)
-+ self.assertSummary(lines[18].text, 'sandbox', '+', ['board4'])
-
- # The second line of errors[3] is a duplicate, so buildman will drop it
- expect = errors[3].rstrip().split('\n')
-diff --git a/tools/buildman/toolchain.py b/tools/buildman/toolchain.py
-index a65737fdf8..cc26e2ede5 100644
---- a/tools/buildman/toolchain.py
-+++ b/tools/buildman/toolchain.py
-@@ -4,18 +4,19 @@
-
- import re
- import glob
--from HTMLParser import HTMLParser
-+from html.parser import HTMLParser
- import os
- import sys
- import tempfile
--import urllib2
-+import urllib.request, urllib.error, urllib.parse
-
- import bsettings
- import command
- import terminal
-+import tools
-
- (PRIORITY_FULL_PREFIX, PRIORITY_PREFIX_GCC, PRIORITY_PREFIX_GCC_PATH,
-- PRIORITY_CALC) = range(4)
-+ PRIORITY_CALC) = list(range(4))
-
- # Simple class to collect links from a page
- class MyHTMLParser(HTMLParser):
-@@ -100,15 +101,15 @@ class Toolchain:
- raise_on_error=False)
- self.ok = result.return_code == 0
- if verbose:
-- print 'Tool chain test: ',
-+ print('Tool chain test: ', end=' ')
- if self.ok:
-- print "OK, arch='%s', priority %d" % (self.arch,
-- self.priority)
-+ print("OK, arch='%s', priority %d" % (self.arch,
-+ self.priority))
- else:
-- print 'BAD'
-- print 'Command: ', cmd
-- print result.stdout
-- print result.stderr
-+ print('BAD')
-+ print('Command: ', cmd)
-+ print(result.stdout)
-+ print(result.stderr)
- else:
- self.ok = True
-
-@@ -138,7 +139,7 @@ class Toolchain:
- value = ''
- for name, value in bsettings.GetItems('toolchain-wrapper'):
- if not value:
-- print "Warning: Wrapper not found"
-+ print("Warning: Wrapper not found")
- if value:
- value = value + ' '
-
-@@ -227,11 +228,11 @@ class Toolchains:
- """
- toolchains = bsettings.GetItems('toolchain')
- if show_warning and not toolchains:
-- print ("Warning: No tool chains. Please run 'buildman "
-+ print(("Warning: No tool chains. Please run 'buildman "
- "--fetch-arch all' to download all available toolchains, or "
- "add a [toolchain] section to your buildman config file "
- "%s. See README for details" %
-- bsettings.config_fname)
-+ bsettings.config_fname))
-
- paths = []
- for name, value in toolchains:
-@@ -272,10 +273,10 @@ class Toolchains:
- if add_it:
- self.toolchains[toolchain.arch] = toolchain
- elif verbose:
-- print ("Toolchain '%s' at priority %d will be ignored because "
-+ print(("Toolchain '%s' at priority %d will be ignored because "
- "another toolchain for arch '%s' has priority %d" %
- (toolchain.gcc, toolchain.priority, toolchain.arch,
-- self.toolchains[toolchain.arch].priority))
-+ self.toolchains[toolchain.arch].priority)))
-
- def ScanPath(self, path, verbose):
- """Scan a path for a valid toolchain
-@@ -289,9 +290,9 @@ class Toolchains:
- fnames = []
- for subdir in ['.', 'bin', 'usr/bin']:
- dirname = os.path.join(path, subdir)
-- if verbose: print " - looking in '%s'" % dirname
-+ if verbose: print(" - looking in '%s'" % dirname)
- for fname in glob.glob(dirname + '/*gcc'):
-- if verbose: print " - found '%s'" % fname
-+ if verbose: print(" - found '%s'" % fname)
- fnames.append(fname)
- return fnames
-
-@@ -321,9 +322,9 @@ class Toolchains:
- Args:
- verbose: True to print out progress information
- """
-- if verbose: print 'Scanning for tool chains'
-+ if verbose: print('Scanning for tool chains')
- for name, value in self.prefixes:
-- if verbose: print " - scanning prefix '%s'" % value
-+ if verbose: print(" - scanning prefix '%s'" % value)
- if os.path.exists(value):
- self.Add(value, True, verbose, PRIORITY_FULL_PREFIX, name)
- continue
-@@ -335,10 +336,10 @@ class Toolchains:
- for f in fname_list:
- self.Add(f, True, verbose, PRIORITY_PREFIX_GCC_PATH, name)
- if not fname_list:
-- raise ValueError, ("No tool chain found for prefix '%s'" %
-+ raise ValueError("No tool chain found for prefix '%s'" %
- value)
- for path in self.paths:
-- if verbose: print " - scanning path '%s'" % path
-+ if verbose: print(" - scanning path '%s'" % path)
- fnames = self.ScanPath(path, verbose)
- for fname in fnames:
- self.Add(fname, True, verbose)
-@@ -346,13 +347,13 @@ class Toolchains:
- def List(self):
- """List out the selected toolchains for each architecture"""
- col = terminal.Color()
-- print col.Color(col.BLUE, 'List of available toolchains (%d):' %
-- len(self.toolchains))
-+ print(col.Color(col.BLUE, 'List of available toolchains (%d):' %
-+ len(self.toolchains)))
- if len(self.toolchains):
-- for key, value in sorted(self.toolchains.iteritems()):
-- print '%-10s: %s' % (key, value.gcc)
-+ for key, value in sorted(self.toolchains.items()):
-+ print('%-10s: %s' % (key, value.gcc))
- else:
-- print 'None'
-+ print('None')
-
- def Select(self, arch):
- """Returns the toolchain for a given architecture
-@@ -370,7 +371,7 @@ class Toolchains:
- return self.toolchains[alias]
-
- if not arch in self.toolchains:
-- raise ValueError, ("No tool chain found for arch '%s'" % arch)
-+ raise ValueError("No tool chain found for arch '%s'" % arch)
- return self.toolchains[arch]
-
- def ResolveReferences(self, var_dict, args):
-@@ -464,9 +465,9 @@ class Toolchains:
- links = []
- for version in versions:
- url = '%s/%s/%s/' % (base, arch, version)
-- print 'Checking: %s' % url
-- response = urllib2.urlopen(url)
-- html = response.read()
-+ print('Checking: %s' % url)
-+ response = urllib.request.urlopen(url)
-+ html = tools.ToString(response.read())
- parser = MyHTMLParser(fetch_arch)
- parser.feed(html)
- if fetch_arch == 'list':
-@@ -488,14 +489,14 @@ class Toolchains:
- Full path to the downloaded archive file in that directory,
- or None if there was an error while downloading
- """
-- print 'Downloading: %s' % url
-+ print('Downloading: %s' % url)
- leaf = url.split('/')[-1]
- tmpdir = tempfile.mkdtemp('.buildman')
-- response = urllib2.urlopen(url)
-+ response = urllib.request.urlopen(url)
- fname = os.path.join(tmpdir, leaf)
- fd = open(fname, 'wb')
- meta = response.info()
-- size = int(meta.getheaders('Content-Length')[0])
-+ size = int(meta.get('Content-Length'))
- done = 0
- block_size = 1 << 16
- status = ''
-@@ -504,19 +505,19 @@ class Toolchains:
- while True:
- buffer = response.read(block_size)
- if not buffer:
-- print chr(8) * (len(status) + 1), '\r',
-+ print(chr(8) * (len(status) + 1), '\r', end=' ')
- break
-
- done += len(buffer)
- fd.write(buffer)
-- status = r'%10d MiB [%3d%%]' % (done / 1024 / 1024,
-- done * 100 / size)
-+ status = r'%10d MiB [%3d%%]' % (done // 1024 // 1024,
-+ done * 100 // size)
- status = status + chr(8) * (len(status) + 1)
-- print status,
-+ print(status, end=' ')
- sys.stdout.flush()
- fd.close()
- if done != size:
-- print 'Error, failed to download'
-+ print('Error, failed to download')
- os.remove(fname)
- fname = None
- return tmpdir, fname
-@@ -565,11 +566,11 @@ class Toolchains:
- """
- # Fist get the URL for this architecture
- col = terminal.Color()
-- print col.Color(col.BLUE, "Downloading toolchain for arch '%s'" % arch)
-+ print(col.Color(col.BLUE, "Downloading toolchain for arch '%s'" % arch))
- url = self.LocateArchUrl(arch)
- if not url:
-- print ("Cannot find toolchain for arch '%s' - use 'list' to list" %
-- arch)
-+ print(("Cannot find toolchain for arch '%s' - use 'list' to list" %
-+ arch))
- return 2
- home = os.environ['HOME']
- dest = os.path.join(home, '.buildman-toolchains')
-@@ -580,28 +581,28 @@ class Toolchains:
- tmpdir, tarfile = self.Download(url)
- if not tarfile:
- return 1
-- print col.Color(col.GREEN, 'Unpacking to: %s' % dest),
-+ print(col.Color(col.GREEN, 'Unpacking to: %s' % dest), end=' ')
- sys.stdout.flush()
- path = self.Unpack(tarfile, dest)
- os.remove(tarfile)
- os.rmdir(tmpdir)
-- print
-+ print()
-
- # Check that the toolchain works
-- print col.Color(col.GREEN, 'Testing')
-+ print(col.Color(col.GREEN, 'Testing'))
- dirpath = os.path.join(dest, path)
- compiler_fname_list = self.ScanPath(dirpath, True)
- if not compiler_fname_list:
-- print 'Could not locate C compiler - fetch failed.'
-+ print('Could not locate C compiler - fetch failed.')
- return 1
- if len(compiler_fname_list) != 1:
-- print col.Color(col.RED, 'Warning, ambiguous toolchains: %s' %
-- ', '.join(compiler_fname_list))
-+ print(col.Color(col.RED, 'Warning, ambiguous toolchains: %s' %
-+ ', '.join(compiler_fname_list)))
- toolchain = Toolchain(compiler_fname_list[0], True, True)
-
- # Make sure that it will be found by buildman
- if not self.TestSettingsHasPath(dirpath):
-- print ("Adding 'download' to config file '%s'" %
-- bsettings.config_fname)
-+ print(("Adding 'download' to config file '%s'" %
-+ bsettings.config_fname))
- bsettings.SetItem('toolchain', 'download', '%s/*/*' % dest)
- return 0
---
-2.24.0
-
+++ /dev/null
-From 3b3e3c0f6c261a8c9f989d437dc261ba84467d4f Mon Sep 17 00:00:00 2001
-From: Simon Glass <sjg@chromium.org>
-Date: Thu, 31 Oct 2019 07:42:50 -0600
-Subject: [PATCH] patman: Adjust 'command' to return strings instead of bytes
-
-At present all the 'command' methods return bytes. Most of the time we
-actually want strings, so change this. We still need to keep the internal
-representation as bytes since otherwise unicode strings might break over
-a read() boundary (e.g. 4KB), causing errors. But we can convert the end
-result to strings.
-
-Add a 'binary' parameter to cover the few cases where bytes are needed.
-
-Upstream-Status: Backport
-
-Signed-off-by: Simon Glass <sjg@chromium.org>
----
- tools/binman/cbfs_util_test.py | 2 +-
- tools/binman/ftest.py | 2 +-
- tools/patman/command.py | 31 +++++++++++++++++++++++--------
- tools/patman/tools.py | 29 +++++++++++++++++++++--------
- 4 files changed, 46 insertions(+), 18 deletions(-)
-
-diff --git a/tools/binman/cbfs_util_test.py b/tools/binman/cbfs_util_test.py
-index 772c794ece..ddc2e09e35 100755
---- a/tools/binman/cbfs_util_test.py
-+++ b/tools/binman/cbfs_util_test.py
-@@ -56,7 +56,7 @@ class TestCbfs(unittest.TestCase):
- cls.have_lz4 = True
- try:
- tools.Run('lz4', '--no-frame-crc', '-c',
-- tools.GetInputFilename('u-boot.bin'))
-+ tools.GetInputFilename('u-boot.bin'), binary=True)
- except:
- cls.have_lz4 = False
-
---- a/tools/binman/ftest.py
-+++ b/tools/binman/ftest.py
-@@ -151,7 +151,7 @@ class TestFunctional(unittest.TestCase):
- self.have_lz4 = True
- try:
- tools.Run('lz4', '--no-frame-crc', '-c',
-- os.path.join(self._indir, 'u-boot.bin'))
-+ os.path.join(self._indir, 'u-boot.bin'), binary=True)
- except:
- self.have_lz4 = False
-
-diff --git a/tools/patman/command.py b/tools/patman/command.py
-index 16299f3f5b..5fbd2c4a3e 100644
---- a/tools/patman/command.py
-+++ b/tools/patman/command.py
-@@ -4,6 +4,7 @@
-
- import os
- import cros_subprocess
-+import tools
-
- """Shell command ease-ups for Python."""
-
-@@ -31,6 +32,13 @@ class CommandResult:
- self.return_code = return_code
- self.exception = exception
-
-+ def ToOutput(self, binary):
-+ if not binary:
-+ self.stdout = tools.ToString(self.stdout)
-+ self.stderr = tools.ToString(self.stderr)
-+ self.combined = tools.ToString(self.combined)
-+ return self
-+
-
- # This permits interception of RunPipe for test purposes. If it is set to
- # a function, then that function is called with the pipe list being
-@@ -41,7 +49,7 @@ test_result = None
-
- def RunPipe(pipe_list, infile=None, outfile=None,
- capture=False, capture_stderr=False, oneline=False,
-- raise_on_error=True, cwd=None, **kwargs):
-+ raise_on_error=True, cwd=None, binary=False, **kwargs):
- """
- Perform a command pipeline, with optional input/output filenames.
-
-@@ -67,7 +75,7 @@ def RunPipe(pipe_list, infile=None, outfile=None,
- else:
- return test_result
- # No result: fall through to normal processing
-- result = CommandResult()
-+ result = CommandResult(b'', b'', b'')
- last_pipe = None
- pipeline = list(pipe_list)
- user_pipestr = '|'.join([' '.join(pipe) for pipe in pipe_list])
-@@ -93,29 +101,36 @@ def RunPipe(pipe_list, infile=None, outfile=None,
- if raise_on_error:
- raise Exception("Error running '%s': %s" % (user_pipestr, str))
- result.return_code = 255
-- return result
-+ return result.ToOutput(binary)
-
- if capture:
- result.stdout, result.stderr, result.combined = (
- last_pipe.CommunicateFilter(None))
- if result.stdout and oneline:
-- result.output = result.stdout.rstrip('\r\n')
-+ result.output = result.stdout.rstrip(b'\r\n')
- result.return_code = last_pipe.wait()
- else:
- result.return_code = os.waitpid(last_pipe.pid, 0)[1]
- if raise_on_error and result.return_code:
- raise Exception("Error running '%s'" % user_pipestr)
-- return result
-+ return result.ToOutput(binary)
-
- def Output(*cmd, **kwargs):
- kwargs['raise_on_error'] = kwargs.get('raise_on_error', True)
- return RunPipe([cmd], capture=True, **kwargs).stdout
-
- def OutputOneLine(*cmd, **kwargs):
-+ """Run a command and output it as a single-line string
-+
-+ The command us expected to produce a single line of output
-+
-+ Returns:
-+ String containing output of command
-+ """
- raise_on_error = kwargs.pop('raise_on_error', True)
-- return (RunPipe([cmd], capture=True, oneline=True,
-- raise_on_error=raise_on_error,
-- **kwargs).stdout.strip())
-+ result = RunPipe([cmd], capture=True, oneline=True,
-+ raise_on_error=raise_on_error, **kwargs).stdout.strip()
-+ return result
-
- def Run(*cmd, **kwargs):
- return RunPipe([cmd], **kwargs).stdout
-diff --git a/tools/patman/tools.py b/tools/patman/tools.py
-index 4a7fcdad21..3feddb292f 100644
---- a/tools/patman/tools.py
-+++ b/tools/patman/tools.py
-@@ -186,7 +186,7 @@ def PathHasFile(path_spec, fname):
- return True
- return False
-
--def Run(name, *args):
-+def Run(name, *args, **kwargs):
- """Run a tool with some arguments
-
- This runs a 'tool', which is a program used by binman to process files and
-@@ -201,13 +201,14 @@ def Run(name, *args):
- CommandResult object
- """
- try:
-+ binary = kwargs.get('binary')
- env = None
- if tool_search_paths:
- env = dict(os.environ)
- env['PATH'] = ':'.join(tool_search_paths) + ':' + env['PATH']
- all_args = (name,) + args
- result = command.RunPipe([all_args], capture=True, capture_stderr=True,
-- env=env, raise_on_error=False)
-+ env=env, raise_on_error=False, binary=binary)
- if result.return_code:
- raise Exception("Error %d running '%s': %s" %
- (result.return_code,' '.join(all_args),
-@@ -375,7 +376,7 @@ def ToBytes(string):
- """Convert a str type into a bytes type
-
- Args:
-- string: string to convert value
-+ string: string to convert
-
- Returns:
- Python 3: A bytes type
-@@ -385,6 +386,18 @@ def ToBytes(string):
- return string.encode('utf-8')
- return string
-
-+def ToString(bval):
-+ """Convert a bytes type into a str type
-+
-+ Args:
-+ bval: bytes value to convert
-+
-+ Returns:
-+ Python 3: A bytes type
-+ Python 2: A string type
-+ """
-+ return bval.decode('utf-8')
-+
- def Compress(indata, algo, with_header=True):
- """Compress some data using a given algorithm
-
-@@ -406,14 +419,14 @@ def Compress(indata, algo, with_header=True):
- fname = GetOutputFilename('%s.comp.tmp' % algo)
- WriteFile(fname, indata)
- if algo == 'lz4':
-- data = Run('lz4', '--no-frame-crc', '-c', fname)
-+ data = Run('lz4', '--no-frame-crc', '-c', fname, binary=True)
- # cbfstool uses a very old version of lzma
- elif algo == 'lzma':
- outfname = GetOutputFilename('%s.comp.otmp' % algo)
- Run('lzma_alone', 'e', fname, outfname, '-lc1', '-lp0', '-pb0', '-d8')
- data = ReadFile(outfname)
- elif algo == 'gzip':
-- data = Run('gzip', '-c', fname)
-+ data = Run('gzip', '-c', fname, binary=True)
- else:
- raise ValueError("Unknown algorithm '%s'" % algo)
- if with_header:
-@@ -446,13 +459,13 @@ def Decompress(indata, algo, with_header=True):
- with open(fname, 'wb') as fd:
- fd.write(indata)
- if algo == 'lz4':
-- data = Run('lz4', '-dc', fname)
-+ data = Run('lz4', '-dc', fname, binary=True)
- elif algo == 'lzma':
- outfname = GetOutputFilename('%s.decomp.otmp' % algo)
- Run('lzma_alone', 'd', fname, outfname)
-- data = ReadFile(outfname)
-+ data = ReadFile(outfname, binary=True)
- elif algo == 'gzip':
-- data = Run('gzip', '-cd', fname)
-+ data = Run('gzip', '-cd', fname, binary=True)
- else:
- raise ValueError("Unknown algorithm '%s'" % algo)
- return data
---
-2.24.0
-
