From 49b5ea23de6460a8866fdefefbdc086fac51eb52 Mon Sep 17 00:00:00 2001 From: Sean Kelly Date: Thu, 22 Sep 2016 13:03:34 -0500 Subject: [PATCH 01/16] Save the Travis configuration for use in the builders Save the Travis configuration so the builders can get options that are to be added. The defaults matrix will have the option to be overriden but it needs to be overriden somewhere and that needs to be available when triggering jobs. --- buildbot_travis/configurator.py | 4 +++- buildbot_travis/steps/create_steps.py | 4 ++++ buildbot_travis/steps/spawner.py | 4 +++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/buildbot_travis/configurator.py b/buildbot_travis/configurator.py index 6f5911d..a8f2c8a 100644 --- a/buildbot_travis/configurator.py +++ b/buildbot_travis/configurator.py @@ -356,7 +356,7 @@ def uniq(tags): # Define the builder for the main job f = factory.BuildFactory() vcsManager.addSourceSteps(f) - f.addStep(TravisSetupSteps()) + f.addStep(TravisSetupSteps(cfgdict=self.cfgdict)) self.config['builders'].append(BuilderConfig( name=job_name, @@ -379,6 +379,7 @@ def uniq(tags): vcsManager.addSourceSteps(f) f.addStep(TravisTrigger( scheduler=job_name, + cfgdict=self.cfgdict, )) properties = dict(TRAVIS_PULL_REQUEST=False) properties.update(self.properties) @@ -431,6 +432,7 @@ def uniq(tags): vcsManager.addSourceSteps(f) f.addStep(TravisTrigger( scheduler=job_name, + cfgdict=self.cfgdict, )) self.config['builders'].append(BuilderConfig( diff --git a/buildbot_travis/steps/create_steps.py b/buildbot_travis/steps/create_steps.py index a80d065..e0e20a0 100644 --- a/buildbot_travis/steps/create_steps.py +++ b/buildbot_travis/steps/create_steps.py @@ -217,6 +217,10 @@ class TravisSetupSteps(ConfigurableStep): MAX_NAME_LENGTH = 47 disable = False + def __init__(self, cfgdict, **kwargs): + self.cfgdict = cfgdict + ConfigurableStep.__init__(self, **kwargs) + def addSetupVirtualEnv(self, python): step = SetupVirtualEnv(python, doStepIf=not self.disable) self.build.addStepsAfterLastStep([step]) diff --git a/buildbot_travis/steps/spawner.py b/buildbot_travis/steps/spawner.py index 99db15a..ba3d524 100644 --- a/buildbot_travis/steps/spawner.py +++ b/buildbot_travis/steps/spawner.py @@ -23,10 +23,12 @@ class TravisTrigger(Trigger, ConfigurableStepMixin): - def __init__(self, scheduler, **kwargs): + + def __init__(self, scheduler, cfgdict, **kwargs): if "name" not in kwargs: kwargs['name'] = 'trigger' self.config = None + self.cfgdict = cfgdict Trigger.__init__( self, waitForFinish=True, From 1119ed9bcab1fef0fc636ce1fe8c8e2b19074e52 Mon Sep 17 00:00:00 2001 From: Sean Kelly Date: Thu, 22 Sep 2016 13:58:48 -0500 Subject: [PATCH 02/16] Pass the Travis configuration to TravisYml TravisYml needs to have the configuration so it can override its defaults when parsing the .travis.yml file. --- buildbot_travis/steps/base.py | 2 +- buildbot_travis/travisyml.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/buildbot_travis/steps/base.py b/buildbot_travis/steps/base.py index 5f9a136..76cb8f9 100644 --- a/buildbot_travis/steps/base.py +++ b/buildbot_travis/steps/base.py @@ -70,7 +70,7 @@ def getStepConfig(self): self.addCompleteLog(filename, travis_yml) - config = TravisYml() + config = TravisYml(self.cfgdict) try: config.parse(travis_yml) except TravisYmlInvalid as e: diff --git a/buildbot_travis/travisyml.py b/buildbot_travis/travisyml.py index 44ed673..875c89f 100644 --- a/buildbot_travis/travisyml.py +++ b/buildbot_travis/travisyml.py @@ -93,7 +93,7 @@ class TravisYml(object): Loads a .travis.yml file and parses it. """ - def __init__(self): + def __init__(self, cfgdict=None): self.language = None self.image = None self.environments = [{}] @@ -105,6 +105,9 @@ def __init__(self): self.email = TravisYmlEmail() self.irc = TravisYmlIrc() self.config = None + self.cfgdict = {} + if cfgdict: + self.cfgdict = cfgdict def parse(self, config_input): try: From 8c480f8c2fb6667726b95c4acf5702b4510d61b6 Mon Sep 17 00:00:00 2001 From: Sean Kelly Date: Wed, 21 Sep 2016 10:04:02 -0500 Subject: [PATCH 03/16] Set a default matrix to use and allow overriding The default matrix lists all of the options for the thing that is being built. The default os and dist match what Travis-CI does, with os set to "linux" and dist set to "precise". The default matrix also sets the keys to search for in the .travis.yml file to expand the build matrix. Each language has different keys about what versions to use. Many use only one, the name of the language, but others have a differently named key or have multiple keys. Support the common, simple case by allowing a language to point to a list or tuple to list the versions to run by default. --- buildbot_travis/travisyml.py | 58 +++++++++++++++++++++++++++++++----- 1 file changed, 50 insertions(+), 8 deletions(-) diff --git a/buildbot_travis/travisyml.py b/buildbot_travis/travisyml.py index 875c89f..6d5d391 100644 --- a/buildbot_travis/travisyml.py +++ b/buildbot_travis/travisyml.py @@ -17,7 +17,9 @@ from __future__ import print_function from future.utils import string_types +import itertools import re +from copy import deepcopy import yaml from buildbot.plugins import util @@ -26,6 +28,18 @@ TRAVIS_HOOKS = ("before_install", "install", "after_install", "before_script", "script", "after_script") +DEFAULT_MATRIX = { + 'os': ( + 'linux', + ), + 'dist': ( + 'precise', + ), + 'language': { + 'python': ('2.7',), + }, +} + class TravisYmlInvalid(Exception): pass @@ -105,6 +119,7 @@ def __init__(self, cfgdict=None): self.email = TravisYmlEmail() self.irc = TravisYmlIrc() self.config = None + self.default_matrix = deepcopy(DEFAULT_MATRIX) self.cfgdict = {} if cfgdict: self.cfgdict = cfgdict @@ -118,6 +133,7 @@ def parse(self, config_input): def parse_dict(self, config): self.config = config + self.load_cfgdict_options() self.parse_language() self.parse_label_mapping() self.parse_envs() @@ -127,6 +143,11 @@ def parse_dict(self, config): self.parse_notifications_email() self.parse_notifications_irc() + def load_cfgdict_options(self): + default_matrix = self.cfgdict.get('default_matrix') + if isinstance(default_matrix, dict): + self.default_matrix.update(default_matrix) + def parse_language(self): try: self.language = self.config['language'] @@ -189,15 +210,36 @@ def parse_branches(self): def parse_matrix(self): matrix = [] - python = self.config.get("python", ["python2.6"]) - if not isinstance(python, list): - python = [python] # First of all, build the implicit matrix - for lang in python: - for env in self.environments: - matrix.append(dict( - python=lang, - env=env, )) + supported_languages = self.default_matrix.get('language', {}) + language_options = supported_languages.get(self.language) + if not isinstance(language_options, (tuple, list)): + language_options = [language_options] + # Many languages use their name as the key to check for versions to use. + if isinstance(language_options, (tuple, list)): + for language_version in self.config.get(self.language, language_options): + for env in self.environments: + matrix.append({self.language: language_version, 'env': env}) + elif isinstance(language_options, dict): + # Get a view of the keys this language supports. Use those + # keys to check if they specified in the config, otherwise + # use the defaults. Do a cross-product across all of the + # keys to get all of the combinations. Finally, zip together + # the keys and the particular combination to convert to a + # dict to populate the matrix. + build_matrix_keys = sorted(list(language_options.keys())) + matrix_versions = [self.config.get(k, language_options[k]) + for k in build_matrix_keys] + # Ensure everything is at least a list of the versions for this + # language. + matrix_versions = [v if isinstance(v, (tuple, list)) else [v] + for v in matrix_versions] + for matrix_combination in itertools.product(*matrix_versions): + for env in self.environments: + lang_matrix = dict(itertools.izip(build_matrix_keys, + matrix_combination)) + lang_matrix['env'] = env + matrix.append(lang_matrix) cfg = self.config.get("matrix", {}) From 3c8813dfe7b20acd8d41a0eba670d887105e991f Mon Sep 17 00:00:00 2001 From: Sean Kelly Date: Wed, 21 Sep 2016 12:46:28 -0500 Subject: [PATCH 04/16] Update tests to match default matrix --- buildbot_travis/tests/test_travisyml.py | 44 +++++++++++++------------ 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/buildbot_travis/tests/test_travisyml.py b/buildbot_travis/tests/test_travisyml.py index a121db1..229aa4b 100644 --- a/buildbot_travis/tests/test_travisyml.py +++ b/buildbot_travis/tests/test_travisyml.py @@ -27,7 +27,9 @@ class TravisYmlTestCase(unittest.TestCase): def setUp(self): self.t = TravisYml() - self.t.config = {} + self.t.config = {'language': 'python'} + self.t.load_cfgdict_options() + self.t.parse_language() class TestYamlParsing(TravisYmlTestCase): @@ -102,7 +104,7 @@ def test_singleenv(self): self.t.parse_matrix() self.assertEqual( - self.t.matrix, [dict(python="python2.6", env=dict(FOO='1', BAR='2')), ]) + self.t.matrix, [dict(python="2.7", env=dict(FOO='1', BAR='2')), ]) def test_multienv(self): self.t.config["env"] = ["FOO=1 BAR=2", "FOO=2 BAR=1"] @@ -112,8 +114,8 @@ def test_multienv(self): self.t.parse_matrix() self.assertEqual(self.t.matrix, [ - dict(python="python2.6", env=dict(FOO='1', BAR='2')), - dict(python="python2.6", env=dict(FOO='2', BAR='1')), + dict(python="2.7", env=dict(FOO='1', BAR='2')), + dict(python="2.7", env=dict(FOO='2', BAR='1')), ]) def test_globalenv(self): @@ -124,8 +126,8 @@ def test_globalenv(self): self.t.parse_matrix() self.assertEqual(self.t.matrix, [ - dict(python="python2.6", env=dict(FOOBAR='0', FOO='1', BAR='2')), - dict(python="python2.6", env=dict(FOOBAR='0', FOO='2', BAR='1')), + dict(python="2.7", env=dict(FOOBAR='0', FOO='1', BAR='2')), + dict(python="2.7", env=dict(FOOBAR='0', FOO='2', BAR='1')), ]) def test_emptymatrixlenv(self): @@ -136,7 +138,7 @@ def test_emptymatrixlenv(self): self.t.parse_matrix() self.assertEqual(self.t.matrix, [ - dict(python="python2.6", env=dict(FOOBAR='0')), + dict(python="2.7", env=dict(FOOBAR='0')), ]) @@ -145,7 +147,7 @@ class TestMatrix(TravisYmlTestCase): def test_exclude_match(self): self.t.config["env"] = ["FOO=1 BAR=2", "FOO=2 BAR=1"] m = self.t.config["matrix"] = {} - m['exclude'] = [dict(python="python2.6", env="FOO=2 BAR=1")] + m['exclude'] = [dict(python="2.7", env="FOO=2 BAR=1")] self.t.parse_envs() self.t.parse_matrix() @@ -157,54 +159,54 @@ def test_exclude_match(self): def test_exclude_subset_match(self): self.t.config["env"] = ["FOO=1 BAR=2", "FOO=2 BAR=1 SPAM=3"] m = self.t.config["matrix"] = {} - m['exclude'] = [dict(python="python2.6", env="FOO=2 BAR=1")] + m['exclude'] = [dict(python="2.7", env="FOO=2 BAR=1")] self.t.parse_envs() self.t.parse_matrix() self.assertEqual(self.t.matrix, [ - dict(python="python2.6", env=dict(FOO='1', BAR='2')), + dict(python="2.7", env=dict(FOO='1', BAR='2')), ]) def test_exclude_nomatch(self): self.t.config["env"] = ["FOO=1 BAR=2", "FOO=2 BAR=1"] m = self.t.config["matrix"] = {} - m['exclude'] = [dict(python="python2.6", env="FOO=2 BAR=3")] + m['exclude'] = [dict(python="2.7", env="FOO=2 BAR=3")] self.t.parse_envs() self.t.parse_matrix() self.assertEqual(self.t.matrix, [ - dict(python="python2.6", env=dict(FOO='1', BAR='2')), - dict(python="python2.6", env=dict(FOO='2', BAR='1')), + dict(python="2.7", env=dict(FOO='1', BAR='2')), + dict(python="2.7", env=dict(FOO='2', BAR='1')), ]) def test_include(self): self.t.config["env"] = ["FOO=1 BAR=2", "FOO=2 BAR=1"] m = self.t.config["matrix"] = {} - m['include'] = [dict(python="python2.6", env="FOO=2 BAR=3")] + m['include'] = [dict(python="2.7", env="FOO=2 BAR=3")] self.t.parse_envs() self.t.parse_matrix() self.assertEqual(self.t.matrix, [ - dict(python="python2.6", env=dict(FOO='1', BAR='2')), - dict(python="python2.6", env=dict(FOO='2', BAR='1')), - dict(python="python2.6", env=dict(FOO='2', BAR='3')), + dict(python="2.7", env=dict(FOO='1', BAR='2')), + dict(python="2.7", env=dict(FOO='2', BAR='1')), + dict(python="2.7", env=dict(FOO='2', BAR='3')), ]) def test_include_with_global(self): self.t.config["env"] = {'global': "CI=true", 'matrix': ["FOO=1 BAR=2", "FOO=2 BAR=1"]} m = self.t.config["matrix"] = {} - m['include'] = [dict(python="python2.6", env="FOO=2 BAR=3")] + m['include'] = [dict(python="2.7", env="FOO=2 BAR=3")] self.t.parse_envs() self.t.parse_matrix() self.assertEqual(self.t.matrix, [ - dict(python="python2.6", env=dict(FOO='1', BAR='2', CI='true')), - dict(python="python2.6", env=dict(FOO='2', BAR='1', CI='true')), - dict(python="python2.6", env=dict(FOO='2', BAR='3', CI='true')), + dict(python="2.7", env=dict(FOO='1', BAR='2', CI='true')), + dict(python="2.7", env=dict(FOO='2', BAR='1', CI='true')), + dict(python="2.7", env=dict(FOO='2', BAR='3', CI='true')), ]) From 490e9fb95572eb31fcc2cfd40c7d0ff97c9b54a6 Mon Sep 17 00:00:00 2001 From: Sean Kelly Date: Mon, 26 Sep 2016 09:40:30 -0500 Subject: [PATCH 05/16] Factor out creating build matrix into own method --- buildbot_travis/travisyml.py | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/buildbot_travis/travisyml.py b/buildbot_travis/travisyml.py index 6d5d391..14d09f5 100644 --- a/buildbot_travis/travisyml.py +++ b/buildbot_travis/travisyml.py @@ -208,18 +208,17 @@ def parse_branches(self): raise TravisYmlInvalid( "'branches' parameter contains neither 'only' nor 'except'") - def parse_matrix(self): + def _build_matrix(self): matrix = [] # First of all, build the implicit matrix supported_languages = self.default_matrix.get('language', {}) language_options = supported_languages.get(self.language) - if not isinstance(language_options, (tuple, list)): + if not isinstance(language_options, (dict, tuple, list)): language_options = [language_options] # Many languages use their name as the key to check for versions to use. if isinstance(language_options, (tuple, list)): for language_version in self.config.get(self.language, language_options): - for env in self.environments: - matrix.append({self.language: language_version, 'env': env}) + matrix.append({self.language: language_version}) elif isinstance(language_options, dict): # Get a view of the keys this language supports. Use those # keys to check if they specified in the config, otherwise @@ -235,11 +234,20 @@ def parse_matrix(self): matrix_versions = [v if isinstance(v, (tuple, list)) else [v] for v in matrix_versions] for matrix_combination in itertools.product(*matrix_versions): - for env in self.environments: - lang_matrix = dict(itertools.izip(build_matrix_keys, - matrix_combination)) - lang_matrix['env'] = env - matrix.append(lang_matrix) + lang_matrix = dict(itertools.izip(build_matrix_keys, + matrix_combination)) + matrix.append(lang_matrix) + + return matrix + + def parse_matrix(self): + build_matrix = self._build_matrix() + matrix = [] + for env in self.environments: + for mat in build_matrix: + mat = mat.copy() + mat['env'] = env + matrix.append(mat) cfg = self.config.get("matrix", {}) From bc31ca28819aa7fc4878a180c6d06b034cce6e47 Mon Sep 17 00:00:00 2001 From: Sean Kelly Date: Mon, 26 Sep 2016 09:49:02 -0500 Subject: [PATCH 06/16] Test build matrix construction --- buildbot_travis/tests/test_travisyml.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/buildbot_travis/tests/test_travisyml.py b/buildbot_travis/tests/test_travisyml.py index 229aa4b..fcd1dd9 100644 --- a/buildbot_travis/tests/test_travisyml.py +++ b/buildbot_travis/tests/test_travisyml.py @@ -142,6 +142,25 @@ def test_emptymatrixlenv(self): ]) +class TestBuildMatrix(TravisYmlTestCase): + + def test_default_language(self): + matrix = self.t._build_matrix() + + self.failUnlessEqual(matrix, [ + dict(python="2.7"), + ]) + + def test_default_multiple_options(self): + self.t.config["python"] = ['2.7', '3.5'] + matrix = self.t._build_matrix() + + self.failUnlessEqual(matrix, [ + dict(python="2.7"), + dict(python="3.5"), + ]) + + class TestMatrix(TravisYmlTestCase): def test_exclude_match(self): From 1976ddbf51635b39fb0cde045d8f474280748dc8 Mon Sep 17 00:00:00 2001 From: Sean Kelly Date: Mon, 26 Sep 2016 10:01:11 -0500 Subject: [PATCH 07/16] Test language with one differently named option --- buildbot_travis/tests/test_travisyml.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/buildbot_travis/tests/test_travisyml.py b/buildbot_travis/tests/test_travisyml.py index fcd1dd9..d9d803f 100644 --- a/buildbot_travis/tests/test_travisyml.py +++ b/buildbot_travis/tests/test_travisyml.py @@ -160,6 +160,21 @@ def test_default_multiple_options(self): dict(python="3.5"), ]) + def test_language_with_dict(self): + self.t.default_matrix = { + 'language': { + 'c': {'compiler': 'gcc'} + } + } + self.t.language = "c" + self.t.config["language"] = "c" + + matrix = self.t._build_matrix() + + self.failUnlessEqual(matrix, [ + dict(compiler='gcc'), + ]) + class TestMatrix(TravisYmlTestCase): From c96230d036796fbb2cc76ec5bd1199bc1893a6b2 Mon Sep 17 00:00:00 2001 From: Sean Kelly Date: Mon, 26 Sep 2016 10:17:14 -0500 Subject: [PATCH 08/16] Test multiple versions with the keys don't match --- buildbot_travis/tests/test_travisyml.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/buildbot_travis/tests/test_travisyml.py b/buildbot_travis/tests/test_travisyml.py index d9d803f..eb371f0 100644 --- a/buildbot_travis/tests/test_travisyml.py +++ b/buildbot_travis/tests/test_travisyml.py @@ -175,6 +175,17 @@ def test_language_with_dict(self): dict(compiler='gcc'), ]) + # Now try again with multiple compilers to use. + self.t.config["compiler"] = ["gcc", "clang", "cc"] + + matrix = self.t._build_matrix() + + self.failUnlessEqual(matrix, [ + dict(compiler='gcc'), + dict(compiler='clang'), + dict(compiler='cc'), + ]) + class TestMatrix(TravisYmlTestCase): From 895dd6e7bbf6ed70a7ba5a889bb324ce2d9f45e2 Mon Sep 17 00:00:00 2001 From: Sean Kelly Date: Mon, 26 Sep 2016 10:22:09 -0500 Subject: [PATCH 09/16] Test language with multiple keys defining the matrix --- buildbot_travis/tests/test_travisyml.py | 55 +++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/buildbot_travis/tests/test_travisyml.py b/buildbot_travis/tests/test_travisyml.py index eb371f0..ddef344 100644 --- a/buildbot_travis/tests/test_travisyml.py +++ b/buildbot_travis/tests/test_travisyml.py @@ -186,6 +186,61 @@ def test_language_with_dict(self): dict(compiler='cc'), ]) + def test_language_multiple_options(self): + self.t.default_matrix = { + 'language': { + 'ruby': { + 'gemfile': 'Gemfile', + 'jdk': 'openjdk7', + 'rvm': '2.2', + } + } + } + self.t.language = "ruby" + self.t.config["language"] = "ruby" + + matrix = self.t._build_matrix() + + self.failUnlessEqual(matrix, [ + dict(gemfile='Gemfile', jdk='openjdk7', rvm='2.2'), + ]) + + # Start exploding the matrix + self.t.config["gemfile"] = ['Gemfile', 'gemfiles/a'] + + matrix = self.t._build_matrix() + + self.failUnlessEqual(matrix, [ + dict(gemfile='Gemfile', jdk='openjdk7', rvm='2.2'), + dict(gemfile='gemfiles/a', jdk='openjdk7', rvm='2.2'), + ]) + + self.t.config["rvm"] = ['2.2', 'jruby'] + + matrix = self.t._build_matrix() + + self.failUnlessEqual(matrix, [ + dict(gemfile='Gemfile', jdk='openjdk7', rvm='2.2'), + dict(gemfile='Gemfile', jdk='openjdk7', rvm='jruby'), + dict(gemfile='gemfiles/a', jdk='openjdk7', rvm='2.2'), + dict(gemfile='gemfiles/a', jdk='openjdk7', rvm='jruby'), + ]) + + self.t.config["jdk"] = ['openjdk7', 'oraclejdk7'] + + matrix = self.t._build_matrix() + + self.failUnlessEqual(matrix, [ + dict(gemfile='Gemfile', jdk='openjdk7', rvm='2.2'), + dict(gemfile='Gemfile', jdk='openjdk7', rvm='jruby'), + dict(gemfile='Gemfile', jdk='oraclejdk7', rvm='2.2'), + dict(gemfile='Gemfile', jdk='oraclejdk7', rvm='jruby'), + dict(gemfile='gemfiles/a', jdk='openjdk7', rvm='2.2'), + dict(gemfile='gemfiles/a', jdk='openjdk7', rvm='jruby'), + dict(gemfile='gemfiles/a', jdk='oraclejdk7', rvm='2.2'), + dict(gemfile='gemfiles/a', jdk='oraclejdk7', rvm='jruby'), + ]) + class TestMatrix(TravisYmlTestCase): From cbb0fa8b863180efbdb54fce59f7cebb9a3c1fcb Mon Sep 17 00:00:00 2001 From: Sean Kelly Date: Mon, 26 Sep 2016 11:01:50 -0500 Subject: [PATCH 10/16] Ensure all values are a dict or list/tuple --- buildbot_travis/travisyml.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/buildbot_travis/travisyml.py b/buildbot_travis/travisyml.py index 14d09f5..7169a31 100644 --- a/buildbot_travis/travisyml.py +++ b/buildbot_travis/travisyml.py @@ -147,6 +147,9 @@ def load_cfgdict_options(self): default_matrix = self.cfgdict.get('default_matrix') if isinstance(default_matrix, dict): self.default_matrix.update(default_matrix) + for k, v in self.default_matrix.iteritems(): + if isinstance(v, basestring): + self.default_matrix[k] = [v] def parse_language(self): try: From d0c74c0ad651211805cab0a3e911a5a942c7bff0 Mon Sep 17 00:00:00 2001 From: Sean Kelly Date: Mon, 26 Sep 2016 11:15:11 -0500 Subject: [PATCH 11/16] Enumerate the os and dist options with the build matrix Expand the overall matrix to build with the given os and dist options from the config. --- buildbot_travis/travisyml.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/buildbot_travis/travisyml.py b/buildbot_travis/travisyml.py index 7169a31..58ecd45 100644 --- a/buildbot_travis/travisyml.py +++ b/buildbot_travis/travisyml.py @@ -243,11 +243,32 @@ def _build_matrix(self): return matrix + def _os_matrix(self, build_matrix): + # The language-level matrix has been built. Merge that with the os and + # dist options from the config. + matrix = [] + os_options = self.config.get('os', self.default_matrix['os']) + if isinstance(os_options, basestring): + os_options = [os_options] + dist_options = self.config.get('dist', self.default_matrix['dist']) + if isinstance(dist_options, basestring): + dist_options = [dist_options] + for os in os_options: + for dist in dist_options: + for build_config in build_matrix: + os_matrix = build_config.copy() + os_matrix['os'] = os + os_matrix['dist'] = dist + matrix.append(os_matrix) + + return matrix + def parse_matrix(self): build_matrix = self._build_matrix() + os_matrix = self._os_matrix(build_matrix) matrix = [] for env in self.environments: - for mat in build_matrix: + for mat in os_matrix: mat = mat.copy() mat['env'] = env matrix.append(mat) From 42d07732d57a64d08289511fbe84a23461cbfd9e Mon Sep 17 00:00:00 2001 From: Sean Kelly Date: Mon, 26 Sep 2016 11:16:00 -0500 Subject: [PATCH 12/16] Test _os_matrix method --- buildbot_travis/tests/test_travisyml.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/buildbot_travis/tests/test_travisyml.py b/buildbot_travis/tests/test_travisyml.py index ddef344..9d904e8 100644 --- a/buildbot_travis/tests/test_travisyml.py +++ b/buildbot_travis/tests/test_travisyml.py @@ -242,6 +242,30 @@ def test_language_multiple_options(self): ]) +class TestOsMatrix(TravisYmlTestCase): + + def test_os_matrix(self): + build_matrix = [dict(python='2.7')] + + matrix = self.t._os_matrix(build_matrix) + + self.failUnlessEqual(matrix, [ + dict(os='linux', dist='precise', python='2.7') + ]) + + def test_multiple_dists(self): + build_matrix = [dict(python='2.7')] + self.t.config["dist"] = ["precise", "trusty", "xenial"] + + matrix = self.t._os_matrix(build_matrix) + + self.failUnlessEqual(matrix, [ + dict(os='linux', dist='precise', python='2.7'), + dict(os='linux', dist='trusty', python='2.7'), + dict(os='linux', dist='xenial', python='2.7'), + ]) + + class TestMatrix(TravisYmlTestCase): def test_exclude_match(self): From 5b71972f1436a55d82e007f71d234f7866562829 Mon Sep 17 00:00:00 2001 From: Sean Kelly Date: Mon, 26 Sep 2016 11:19:33 -0500 Subject: [PATCH 13/16] Add os and dist to every test Now everything testing the matrix needs the os and dist keys set. --- buildbot_travis/tests/test_travisyml.py | 28 ++++++++++++------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/buildbot_travis/tests/test_travisyml.py b/buildbot_travis/tests/test_travisyml.py index 9d904e8..575d713 100644 --- a/buildbot_travis/tests/test_travisyml.py +++ b/buildbot_travis/tests/test_travisyml.py @@ -104,7 +104,7 @@ def test_singleenv(self): self.t.parse_matrix() self.assertEqual( - self.t.matrix, [dict(python="2.7", env=dict(FOO='1', BAR='2')), ]) + self.t.matrix, [dict(python="2.7", env=dict(FOO='1', BAR='2'), os='linux', dist='precise'), ]) def test_multienv(self): self.t.config["env"] = ["FOO=1 BAR=2", "FOO=2 BAR=1"] @@ -114,8 +114,8 @@ def test_multienv(self): self.t.parse_matrix() self.assertEqual(self.t.matrix, [ - dict(python="2.7", env=dict(FOO='1', BAR='2')), - dict(python="2.7", env=dict(FOO='2', BAR='1')), + dict(python="2.7", env=dict(FOO='1', BAR='2'), os='linux', dist='precise'), + dict(python="2.7", env=dict(FOO='2', BAR='1'), os='linux', dist='precise'), ]) def test_globalenv(self): @@ -126,8 +126,8 @@ def test_globalenv(self): self.t.parse_matrix() self.assertEqual(self.t.matrix, [ - dict(python="2.7", env=dict(FOOBAR='0', FOO='1', BAR='2')), - dict(python="2.7", env=dict(FOOBAR='0', FOO='2', BAR='1')), + dict(python="2.7", env=dict(FOOBAR='0', FOO='1', BAR='2'), os='linux', dist='precise'), + dict(python="2.7", env=dict(FOOBAR='0', FOO='2', BAR='1'), os='linux', dist='precise'), ]) def test_emptymatrixlenv(self): @@ -138,7 +138,7 @@ def test_emptymatrixlenv(self): self.t.parse_matrix() self.assertEqual(self.t.matrix, [ - dict(python="2.7", env=dict(FOOBAR='0')), + dict(python="2.7", env=dict(FOOBAR='0'), os='linux', dist='precise'), ]) @@ -277,7 +277,7 @@ def test_exclude_match(self): self.t.parse_matrix() self.assertEqual(self.t.matrix, [ - dict(python="python2.6", env=dict(FOO='1', BAR='2')), + dict(python="2.7", env=dict(FOO='1', BAR='2'), os='linux', dist='precise'), ]) def test_exclude_subset_match(self): @@ -289,7 +289,7 @@ def test_exclude_subset_match(self): self.t.parse_matrix() self.assertEqual(self.t.matrix, [ - dict(python="2.7", env=dict(FOO='1', BAR='2')), + dict(python="2.7", env=dict(FOO='1', BAR='2'), os='linux', dist='precise'), ]) def test_exclude_nomatch(self): @@ -301,8 +301,8 @@ def test_exclude_nomatch(self): self.t.parse_matrix() self.assertEqual(self.t.matrix, [ - dict(python="2.7", env=dict(FOO='1', BAR='2')), - dict(python="2.7", env=dict(FOO='2', BAR='1')), + dict(python="2.7", env=dict(FOO='1', BAR='2'), os='linux', dist='precise'), + dict(python="2.7", env=dict(FOO='2', BAR='1'), os='linux', dist='precise'), ]) def test_include(self): @@ -314,8 +314,8 @@ def test_include(self): self.t.parse_matrix() self.assertEqual(self.t.matrix, [ - dict(python="2.7", env=dict(FOO='1', BAR='2')), - dict(python="2.7", env=dict(FOO='2', BAR='1')), + dict(python="2.7", env=dict(FOO='1', BAR='2'), os='linux', dist='precise'), + dict(python="2.7", env=dict(FOO='2', BAR='1'), os='linux', dist='precise'), dict(python="2.7", env=dict(FOO='2', BAR='3')), ]) @@ -328,8 +328,8 @@ def test_include_with_global(self): self.t.parse_matrix() self.assertEqual(self.t.matrix, [ - dict(python="2.7", env=dict(FOO='1', BAR='2', CI='true')), - dict(python="2.7", env=dict(FOO='2', BAR='1', CI='true')), + dict(python="2.7", env=dict(FOO='1', BAR='2', CI='true'), os='linux', dist='precise'), + dict(python="2.7", env=dict(FOO='2', BAR='1', CI='true'), os='linux', dist='precise'), dict(python="2.7", env=dict(FOO='2', BAR='3', CI='true')), ]) From 69df2040f6f71928dab843681f46e2d789195ffe Mon Sep 17 00:00:00 2001 From: Sean Kelly Date: Mon, 26 Sep 2016 15:36:24 -0500 Subject: [PATCH 14/16] Include the language in the properties --- buildbot_travis/travisyml.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/buildbot_travis/travisyml.py b/buildbot_travis/travisyml.py index 58ecd45..25d7e02 100644 --- a/buildbot_travis/travisyml.py +++ b/buildbot_travis/travisyml.py @@ -221,7 +221,8 @@ def _build_matrix(self): # Many languages use their name as the key to check for versions to use. if isinstance(language_options, (tuple, list)): for language_version in self.config.get(self.language, language_options): - matrix.append({self.language: language_version}) + matrix.append({'language': self.language, + self.language: language_version}) elif isinstance(language_options, dict): # Get a view of the keys this language supports. Use those # keys to check if they specified in the config, otherwise @@ -239,6 +240,7 @@ def _build_matrix(self): for matrix_combination in itertools.product(*matrix_versions): lang_matrix = dict(itertools.izip(build_matrix_keys, matrix_combination)) + lang_matrix['language'] = self.language matrix.append(lang_matrix) return matrix From e84ee401ec59514c8e63256bcc2795a4b0855a66 Mon Sep 17 00:00:00 2001 From: Sean Kelly Date: Mon, 26 Sep 2016 15:48:00 -0500 Subject: [PATCH 15/16] Add language to the tests --- buildbot_travis/tests/test_travisyml.py | 98 ++++++++++++++----------- 1 file changed, 56 insertions(+), 42 deletions(-) diff --git a/buildbot_travis/tests/test_travisyml.py b/buildbot_travis/tests/test_travisyml.py index 575d713..a7573bf 100644 --- a/buildbot_travis/tests/test_travisyml.py +++ b/buildbot_travis/tests/test_travisyml.py @@ -104,7 +104,8 @@ def test_singleenv(self): self.t.parse_matrix() self.assertEqual( - self.t.matrix, [dict(python="2.7", env=dict(FOO='1', BAR='2'), os='linux', dist='precise'), ]) + self.t.matrix, [dict(python="2.7", env=dict(FOO='1', BAR='2'), + os='linux', dist='precise', language='python'), ]) def test_multienv(self): self.t.config["env"] = ["FOO=1 BAR=2", "FOO=2 BAR=1"] @@ -114,8 +115,10 @@ def test_multienv(self): self.t.parse_matrix() self.assertEqual(self.t.matrix, [ - dict(python="2.7", env=dict(FOO='1', BAR='2'), os='linux', dist='precise'), - dict(python="2.7", env=dict(FOO='2', BAR='1'), os='linux', dist='precise'), + dict(python="2.7", env=dict(FOO='1', BAR='2'), os='linux', + dist='precise', language='python'), + dict(python="2.7", env=dict(FOO='2', BAR='1'), os='linux', + dist='precise', language='python'), ]) def test_globalenv(self): @@ -126,8 +129,10 @@ def test_globalenv(self): self.t.parse_matrix() self.assertEqual(self.t.matrix, [ - dict(python="2.7", env=dict(FOOBAR='0', FOO='1', BAR='2'), os='linux', dist='precise'), - dict(python="2.7", env=dict(FOOBAR='0', FOO='2', BAR='1'), os='linux', dist='precise'), + dict(python="2.7", env=dict(FOOBAR='0', FOO='1', BAR='2'), + os='linux', dist='precise', language='python'), + dict(python="2.7", env=dict(FOOBAR='0', FOO='2', BAR='1'), + os='linux', dist='precise', language='python'), ]) def test_emptymatrixlenv(self): @@ -138,7 +143,8 @@ def test_emptymatrixlenv(self): self.t.parse_matrix() self.assertEqual(self.t.matrix, [ - dict(python="2.7", env=dict(FOOBAR='0'), os='linux', dist='precise'), + dict(python="2.7", env=dict(FOOBAR='0'), os='linux', + dist='precise', language='python'), ]) @@ -148,7 +154,7 @@ def test_default_language(self): matrix = self.t._build_matrix() self.failUnlessEqual(matrix, [ - dict(python="2.7"), + dict(language='python', python="2.7"), ]) def test_default_multiple_options(self): @@ -156,8 +162,8 @@ def test_default_multiple_options(self): matrix = self.t._build_matrix() self.failUnlessEqual(matrix, [ - dict(python="2.7"), - dict(python="3.5"), + dict(language='python', python="2.7"), + dict(language='python', python="3.5"), ]) def test_language_with_dict(self): @@ -172,7 +178,7 @@ def test_language_with_dict(self): matrix = self.t._build_matrix() self.failUnlessEqual(matrix, [ - dict(compiler='gcc'), + dict(compiler='gcc', language='c'), ]) # Now try again with multiple compilers to use. @@ -181,9 +187,9 @@ def test_language_with_dict(self): matrix = self.t._build_matrix() self.failUnlessEqual(matrix, [ - dict(compiler='gcc'), - dict(compiler='clang'), - dict(compiler='cc'), + dict(compiler='gcc', language='c'), + dict(compiler='clang', language='c'), + dict(compiler='cc', language='c'), ]) def test_language_multiple_options(self): @@ -202,7 +208,7 @@ def test_language_multiple_options(self): matrix = self.t._build_matrix() self.failUnlessEqual(matrix, [ - dict(gemfile='Gemfile', jdk='openjdk7', rvm='2.2'), + dict(gemfile='Gemfile', jdk='openjdk7', rvm='2.2', language='ruby'), ]) # Start exploding the matrix @@ -211,8 +217,8 @@ def test_language_multiple_options(self): matrix = self.t._build_matrix() self.failUnlessEqual(matrix, [ - dict(gemfile='Gemfile', jdk='openjdk7', rvm='2.2'), - dict(gemfile='gemfiles/a', jdk='openjdk7', rvm='2.2'), + dict(gemfile='Gemfile', jdk='openjdk7', rvm='2.2', language='ruby'), + dict(gemfile='gemfiles/a', jdk='openjdk7', rvm='2.2', language='ruby'), ]) self.t.config["rvm"] = ['2.2', 'jruby'] @@ -220,10 +226,10 @@ def test_language_multiple_options(self): matrix = self.t._build_matrix() self.failUnlessEqual(matrix, [ - dict(gemfile='Gemfile', jdk='openjdk7', rvm='2.2'), - dict(gemfile='Gemfile', jdk='openjdk7', rvm='jruby'), - dict(gemfile='gemfiles/a', jdk='openjdk7', rvm='2.2'), - dict(gemfile='gemfiles/a', jdk='openjdk7', rvm='jruby'), + dict(gemfile='Gemfile', jdk='openjdk7', rvm='2.2', language='ruby'), + dict(gemfile='Gemfile', jdk='openjdk7', rvm='jruby', language='ruby'), + dict(gemfile='gemfiles/a', jdk='openjdk7', rvm='2.2', language='ruby'), + dict(gemfile='gemfiles/a', jdk='openjdk7', rvm='jruby', language='ruby'), ]) self.t.config["jdk"] = ['openjdk7', 'oraclejdk7'] @@ -231,38 +237,38 @@ def test_language_multiple_options(self): matrix = self.t._build_matrix() self.failUnlessEqual(matrix, [ - dict(gemfile='Gemfile', jdk='openjdk7', rvm='2.2'), - dict(gemfile='Gemfile', jdk='openjdk7', rvm='jruby'), - dict(gemfile='Gemfile', jdk='oraclejdk7', rvm='2.2'), - dict(gemfile='Gemfile', jdk='oraclejdk7', rvm='jruby'), - dict(gemfile='gemfiles/a', jdk='openjdk7', rvm='2.2'), - dict(gemfile='gemfiles/a', jdk='openjdk7', rvm='jruby'), - dict(gemfile='gemfiles/a', jdk='oraclejdk7', rvm='2.2'), - dict(gemfile='gemfiles/a', jdk='oraclejdk7', rvm='jruby'), + dict(gemfile='Gemfile', jdk='openjdk7', rvm='2.2', language='ruby'), + dict(gemfile='Gemfile', jdk='openjdk7', rvm='jruby', language='ruby'), + dict(gemfile='Gemfile', jdk='oraclejdk7', rvm='2.2', language='ruby'), + dict(gemfile='Gemfile', jdk='oraclejdk7', rvm='jruby', language='ruby'), + dict(gemfile='gemfiles/a', jdk='openjdk7', rvm='2.2', language='ruby'), + dict(gemfile='gemfiles/a', jdk='openjdk7', rvm='jruby', language='ruby'), + dict(gemfile='gemfiles/a', jdk='oraclejdk7', rvm='2.2', language='ruby'), + dict(gemfile='gemfiles/a', jdk='oraclejdk7', rvm='jruby', language='ruby'), ]) class TestOsMatrix(TravisYmlTestCase): def test_os_matrix(self): - build_matrix = [dict(python='2.7')] + build_matrix = [dict(language='python', python='2.7')] matrix = self.t._os_matrix(build_matrix) self.failUnlessEqual(matrix, [ - dict(os='linux', dist='precise', python='2.7') + dict(os='linux', dist='precise', language='python', python='2.7') ]) def test_multiple_dists(self): - build_matrix = [dict(python='2.7')] + build_matrix = [dict(language='python', python='2.7')] self.t.config["dist"] = ["precise", "trusty", "xenial"] matrix = self.t._os_matrix(build_matrix) self.failUnlessEqual(matrix, [ - dict(os='linux', dist='precise', python='2.7'), - dict(os='linux', dist='trusty', python='2.7'), - dict(os='linux', dist='xenial', python='2.7'), + dict(os='linux', dist='precise', language='python', python='2.7'), + dict(os='linux', dist='trusty', language='python', python='2.7'), + dict(os='linux', dist='xenial', language='python', python='2.7'), ]) @@ -277,7 +283,8 @@ def test_exclude_match(self): self.t.parse_matrix() self.assertEqual(self.t.matrix, [ - dict(python="2.7", env=dict(FOO='1', BAR='2'), os='linux', dist='precise'), + dict(python="2.7", env=dict(FOO='1', BAR='2'), os='linux', + dist='precise', language='python'), ]) def test_exclude_subset_match(self): @@ -289,7 +296,8 @@ def test_exclude_subset_match(self): self.t.parse_matrix() self.assertEqual(self.t.matrix, [ - dict(python="2.7", env=dict(FOO='1', BAR='2'), os='linux', dist='precise'), + dict(python="2.7", env=dict(FOO='1', BAR='2'), os='linux', + dist='precise', language='python'), ]) def test_exclude_nomatch(self): @@ -301,8 +309,10 @@ def test_exclude_nomatch(self): self.t.parse_matrix() self.assertEqual(self.t.matrix, [ - dict(python="2.7", env=dict(FOO='1', BAR='2'), os='linux', dist='precise'), - dict(python="2.7", env=dict(FOO='2', BAR='1'), os='linux', dist='precise'), + dict(python="2.7", env=dict(FOO='1', BAR='2'), os='linux', + dist='precise', language='python'), + dict(python="2.7", env=dict(FOO='2', BAR='1'), os='linux', + dist='precise', language='python'), ]) def test_include(self): @@ -314,8 +324,10 @@ def test_include(self): self.t.parse_matrix() self.assertEqual(self.t.matrix, [ - dict(python="2.7", env=dict(FOO='1', BAR='2'), os='linux', dist='precise'), - dict(python="2.7", env=dict(FOO='2', BAR='1'), os='linux', dist='precise'), + dict(python="2.7", env=dict(FOO='1', BAR='2'), os='linux', + dist='precise', language='python'), + dict(python="2.7", env=dict(FOO='2', BAR='1'), os='linux', + dist='precise', language='python'), dict(python="2.7", env=dict(FOO='2', BAR='3')), ]) @@ -328,8 +340,10 @@ def test_include_with_global(self): self.t.parse_matrix() self.assertEqual(self.t.matrix, [ - dict(python="2.7", env=dict(FOO='1', BAR='2', CI='true'), os='linux', dist='precise'), - dict(python="2.7", env=dict(FOO='2', BAR='1', CI='true'), os='linux', dist='precise'), + dict(python="2.7", env=dict(FOO='1', BAR='2', CI='true'), + os='linux', dist='precise', language='python'), + dict(python="2.7", env=dict(FOO='2', BAR='1', CI='true'), + os='linux', dist='precise', language='python'), dict(python="2.7", env=dict(FOO='2', BAR='3', CI='true')), ]) From 1faa09a4585a46c37e681dd861122e1d15babb50 Mon Sep 17 00:00:00 2001 From: Sean Kelly Date: Wed, 26 Apr 2017 10:23:53 -0500 Subject: [PATCH 16/16] Add documentation to the README --- README.rst | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/README.rst b/README.rst index 859f80e..ca4df21 100644 --- a/README.rst +++ b/README.rst @@ -470,6 +470,30 @@ Here are the 5 steps to setup a Deployment dashboard in Buildbot Travis. ${stage} is the retrieved from the Deployment dashboard. ${version} is retrieved from the Deployment dashboard. +Configuring Travis Defaults +=========================== + +The YAML file or Python dict passed to ``TravisConfigurator`` supports a few keys to set some environment defaults. + +Default Matrix +-------------- +The ``default_matrix`` key contains the default values for any keys the repository's ``.travis.yml`` does not specify. + +Example:: + + default_matrix: + os: linux + dist: debian_7 + language: + python: 2.7 + c: + compiler: gcc + c++: + compiler: g++ + +This example sets the default ``os`` to ``linux``, the default ``dist`` to ``debian_7``, and sets default values for three languages. +If the ``.travis.yml`` has ``language: c``, then it will have ``compiler`` set to ``gcc``. + How it works ============