diff --git a/README.rst b/README.rst index 859f80e..ca4df21 100644 --- a/README.rst +++ b/README.rst @@ -470,6 +470,30 @@ Here are the 5 steps to setup a Deployment dashboard in Buildbot Travis. ${stage} is the retrieved from the Deployment dashboard. ${version} is retrieved from the Deployment dashboard. +Configuring Travis Defaults +=========================== + +The YAML file or Python dict passed to ``TravisConfigurator`` supports a few keys to set some environment defaults. + +Default Matrix +-------------- +The ``default_matrix`` key contains the default values for any keys the repository's ``.travis.yml`` does not specify. + +Example:: + + default_matrix: + os: linux + dist: debian_7 + language: + python: 2.7 + c: + compiler: gcc + c++: + compiler: g++ + +This example sets the default ``os`` to ``linux``, the default ``dist`` to ``debian_7``, and sets default values for three languages. +If the ``.travis.yml`` has ``language: c``, then it will have ``compiler`` set to ``gcc``. + How it works ============ diff --git a/buildbot_travis/configurator.py b/buildbot_travis/configurator.py index 6f5911d..a8f2c8a 100644 --- a/buildbot_travis/configurator.py +++ b/buildbot_travis/configurator.py @@ -356,7 +356,7 @@ def uniq(tags): # Define the builder for the main job f = factory.BuildFactory() vcsManager.addSourceSteps(f) - f.addStep(TravisSetupSteps()) + f.addStep(TravisSetupSteps(cfgdict=self.cfgdict)) self.config['builders'].append(BuilderConfig( name=job_name, @@ -379,6 +379,7 @@ def uniq(tags): vcsManager.addSourceSteps(f) f.addStep(TravisTrigger( scheduler=job_name, + cfgdict=self.cfgdict, )) properties = dict(TRAVIS_PULL_REQUEST=False) properties.update(self.properties) @@ -431,6 +432,7 @@ def uniq(tags): vcsManager.addSourceSteps(f) f.addStep(TravisTrigger( scheduler=job_name, + cfgdict=self.cfgdict, )) self.config['builders'].append(BuilderConfig( diff --git a/buildbot_travis/steps/base.py b/buildbot_travis/steps/base.py index 5f9a136..76cb8f9 100644 --- a/buildbot_travis/steps/base.py +++ b/buildbot_travis/steps/base.py @@ -70,7 +70,7 @@ def getStepConfig(self): self.addCompleteLog(filename, travis_yml) - config = TravisYml() + config = TravisYml(self.cfgdict) try: config.parse(travis_yml) except TravisYmlInvalid as e: diff --git a/buildbot_travis/steps/create_steps.py b/buildbot_travis/steps/create_steps.py index a80d065..e0e20a0 100644 --- a/buildbot_travis/steps/create_steps.py +++ b/buildbot_travis/steps/create_steps.py @@ -217,6 +217,10 @@ class TravisSetupSteps(ConfigurableStep): MAX_NAME_LENGTH = 47 disable = False + def __init__(self, cfgdict, **kwargs): + self.cfgdict = cfgdict + ConfigurableStep.__init__(self, **kwargs) + def addSetupVirtualEnv(self, python): step = SetupVirtualEnv(python, doStepIf=not self.disable) self.build.addStepsAfterLastStep([step]) diff --git a/buildbot_travis/steps/spawner.py b/buildbot_travis/steps/spawner.py index 99db15a..ba3d524 100644 --- a/buildbot_travis/steps/spawner.py +++ b/buildbot_travis/steps/spawner.py @@ -23,10 +23,12 @@ class TravisTrigger(Trigger, ConfigurableStepMixin): - def __init__(self, scheduler, **kwargs): + + def __init__(self, scheduler, cfgdict, **kwargs): if "name" not in kwargs: kwargs['name'] = 'trigger' self.config = None + self.cfgdict = cfgdict Trigger.__init__( self, waitForFinish=True, diff --git a/buildbot_travis/tests/test_travisyml.py b/buildbot_travis/tests/test_travisyml.py index a121db1..a7573bf 100644 --- a/buildbot_travis/tests/test_travisyml.py +++ b/buildbot_travis/tests/test_travisyml.py @@ -27,7 +27,9 @@ class TravisYmlTestCase(unittest.TestCase): def setUp(self): self.t = TravisYml() - self.t.config = {} + self.t.config = {'language': 'python'} + self.t.load_cfgdict_options() + self.t.parse_language() class TestYamlParsing(TravisYmlTestCase): @@ -102,7 +104,8 @@ def test_singleenv(self): self.t.parse_matrix() self.assertEqual( - self.t.matrix, [dict(python="python2.6", env=dict(FOO='1', BAR='2')), ]) + self.t.matrix, [dict(python="2.7", env=dict(FOO='1', BAR='2'), + os='linux', dist='precise', language='python'), ]) def test_multienv(self): self.t.config["env"] = ["FOO=1 BAR=2", "FOO=2 BAR=1"] @@ -112,8 +115,10 @@ def test_multienv(self): self.t.parse_matrix() self.assertEqual(self.t.matrix, [ - dict(python="python2.6", env=dict(FOO='1', BAR='2')), - dict(python="python2.6", env=dict(FOO='2', BAR='1')), + dict(python="2.7", env=dict(FOO='1', BAR='2'), os='linux', + dist='precise', language='python'), + dict(python="2.7", env=dict(FOO='2', BAR='1'), os='linux', + dist='precise', language='python'), ]) def test_globalenv(self): @@ -124,8 +129,10 @@ def test_globalenv(self): self.t.parse_matrix() self.assertEqual(self.t.matrix, [ - dict(python="python2.6", env=dict(FOOBAR='0', FOO='1', BAR='2')), - dict(python="python2.6", env=dict(FOOBAR='0', FOO='2', BAR='1')), + dict(python="2.7", env=dict(FOOBAR='0', FOO='1', BAR='2'), + os='linux', dist='precise', language='python'), + dict(python="2.7", env=dict(FOOBAR='0', FOO='2', BAR='1'), + os='linux', dist='precise', language='python'), ]) def test_emptymatrixlenv(self): @@ -136,7 +143,132 @@ def test_emptymatrixlenv(self): self.t.parse_matrix() self.assertEqual(self.t.matrix, [ - dict(python="python2.6", env=dict(FOOBAR='0')), + dict(python="2.7", env=dict(FOOBAR='0'), os='linux', + dist='precise', language='python'), + ]) + + +class TestBuildMatrix(TravisYmlTestCase): + + def test_default_language(self): + matrix = self.t._build_matrix() + + self.failUnlessEqual(matrix, [ + dict(language='python', python="2.7"), + ]) + + def test_default_multiple_options(self): + self.t.config["python"] = ['2.7', '3.5'] + matrix = self.t._build_matrix() + + self.failUnlessEqual(matrix, [ + dict(language='python', python="2.7"), + dict(language='python', python="3.5"), + ]) + + def test_language_with_dict(self): + self.t.default_matrix = { + 'language': { + 'c': {'compiler': 'gcc'} + } + } + self.t.language = "c" + self.t.config["language"] = "c" + + matrix = self.t._build_matrix() + + self.failUnlessEqual(matrix, [ + dict(compiler='gcc', language='c'), + ]) + + # Now try again with multiple compilers to use. + self.t.config["compiler"] = ["gcc", "clang", "cc"] + + matrix = self.t._build_matrix() + + self.failUnlessEqual(matrix, [ + dict(compiler='gcc', language='c'), + dict(compiler='clang', language='c'), + dict(compiler='cc', language='c'), + ]) + + def test_language_multiple_options(self): + self.t.default_matrix = { + 'language': { + 'ruby': { + 'gemfile': 'Gemfile', + 'jdk': 'openjdk7', + 'rvm': '2.2', + } + } + } + self.t.language = "ruby" + self.t.config["language"] = "ruby" + + matrix = self.t._build_matrix() + + self.failUnlessEqual(matrix, [ + dict(gemfile='Gemfile', jdk='openjdk7', rvm='2.2', language='ruby'), + ]) + + # Start exploding the matrix + self.t.config["gemfile"] = ['Gemfile', 'gemfiles/a'] + + matrix = self.t._build_matrix() + + self.failUnlessEqual(matrix, [ + dict(gemfile='Gemfile', jdk='openjdk7', rvm='2.2', language='ruby'), + dict(gemfile='gemfiles/a', jdk='openjdk7', rvm='2.2', language='ruby'), + ]) + + self.t.config["rvm"] = ['2.2', 'jruby'] + + matrix = self.t._build_matrix() + + self.failUnlessEqual(matrix, [ + dict(gemfile='Gemfile', jdk='openjdk7', rvm='2.2', language='ruby'), + dict(gemfile='Gemfile', jdk='openjdk7', rvm='jruby', language='ruby'), + dict(gemfile='gemfiles/a', jdk='openjdk7', rvm='2.2', language='ruby'), + dict(gemfile='gemfiles/a', jdk='openjdk7', rvm='jruby', language='ruby'), + ]) + + self.t.config["jdk"] = ['openjdk7', 'oraclejdk7'] + + matrix = self.t._build_matrix() + + self.failUnlessEqual(matrix, [ + dict(gemfile='Gemfile', jdk='openjdk7', rvm='2.2', language='ruby'), + dict(gemfile='Gemfile', jdk='openjdk7', rvm='jruby', language='ruby'), + dict(gemfile='Gemfile', jdk='oraclejdk7', rvm='2.2', language='ruby'), + dict(gemfile='Gemfile', jdk='oraclejdk7', rvm='jruby', language='ruby'), + dict(gemfile='gemfiles/a', jdk='openjdk7', rvm='2.2', language='ruby'), + dict(gemfile='gemfiles/a', jdk='openjdk7', rvm='jruby', language='ruby'), + dict(gemfile='gemfiles/a', jdk='oraclejdk7', rvm='2.2', language='ruby'), + dict(gemfile='gemfiles/a', jdk='oraclejdk7', rvm='jruby', language='ruby'), + ]) + + +class TestOsMatrix(TravisYmlTestCase): + + def test_os_matrix(self): + build_matrix = [dict(language='python', python='2.7')] + + matrix = self.t._os_matrix(build_matrix) + + self.failUnlessEqual(matrix, [ + dict(os='linux', dist='precise', language='python', python='2.7') + ]) + + def test_multiple_dists(self): + build_matrix = [dict(language='python', python='2.7')] + self.t.config["dist"] = ["precise", "trusty", "xenial"] + + matrix = self.t._os_matrix(build_matrix) + + self.failUnlessEqual(matrix, [ + dict(os='linux', dist='precise', language='python', python='2.7'), + dict(os='linux', dist='trusty', language='python', python='2.7'), + dict(os='linux', dist='xenial', language='python', python='2.7'), ]) @@ -145,66 +277,74 @@ class TestMatrix(TravisYmlTestCase): def test_exclude_match(self): self.t.config["env"] = ["FOO=1 BAR=2", "FOO=2 BAR=1"] m = self.t.config["matrix"] = {} - m['exclude'] = [dict(python="python2.6", env="FOO=2 BAR=1")] + m['exclude'] = [dict(python="2.7", env="FOO=2 BAR=1")] self.t.parse_envs() self.t.parse_matrix() self.assertEqual(self.t.matrix, [ - dict(python="python2.6", env=dict(FOO='1', BAR='2')), + dict(python="2.7", env=dict(FOO='1', BAR='2'), os='linux', + dist='precise', language='python'), ]) def test_exclude_subset_match(self): self.t.config["env"] = ["FOO=1 BAR=2", "FOO=2 BAR=1 SPAM=3"] m = self.t.config["matrix"] = {} - m['exclude'] = [dict(python="python2.6", env="FOO=2 BAR=1")] + m['exclude'] = [dict(python="2.7", env="FOO=2 BAR=1")] self.t.parse_envs() self.t.parse_matrix() self.assertEqual(self.t.matrix, [ - dict(python="python2.6", env=dict(FOO='1', BAR='2')), + dict(python="2.7", env=dict(FOO='1', BAR='2'), os='linux', + dist='precise', language='python'), ]) def test_exclude_nomatch(self): self.t.config["env"] = ["FOO=1 BAR=2", "FOO=2 BAR=1"] m = self.t.config["matrix"] = {} - m['exclude'] = [dict(python="python2.6", env="FOO=2 BAR=3")] + m['exclude'] = [dict(python="2.7", env="FOO=2 BAR=3")] self.t.parse_envs() self.t.parse_matrix() self.assertEqual(self.t.matrix, [ - dict(python="python2.6", env=dict(FOO='1', BAR='2')), - dict(python="python2.6", env=dict(FOO='2', BAR='1')), + dict(python="2.7", env=dict(FOO='1', BAR='2'), os='linux', + dist='precise', language='python'), + dict(python="2.7", env=dict(FOO='2', BAR='1'), os='linux', + dist='precise', language='python'), ]) def test_include(self): self.t.config["env"] = ["FOO=1 BAR=2", "FOO=2 BAR=1"] m = self.t.config["matrix"] = {} - m['include'] = [dict(python="python2.6", env="FOO=2 BAR=3")] + m['include'] = [dict(python="2.7", env="FOO=2 BAR=3")] self.t.parse_envs() self.t.parse_matrix() self.assertEqual(self.t.matrix, [ - dict(python="python2.6", env=dict(FOO='1', BAR='2')), - dict(python="python2.6", env=dict(FOO='2', BAR='1')), - dict(python="python2.6", env=dict(FOO='2', BAR='3')), + dict(python="2.7", env=dict(FOO='1', BAR='2'), os='linux', + dist='precise', language='python'), + dict(python="2.7", env=dict(FOO='2', BAR='1'), os='linux', + dist='precise', language='python'), + dict(python="2.7", env=dict(FOO='2', BAR='3')), ]) def test_include_with_global(self): self.t.config["env"] = {'global': "CI=true", 'matrix': ["FOO=1 BAR=2", "FOO=2 BAR=1"]} m = self.t.config["matrix"] = {} - m['include'] = [dict(python="python2.6", env="FOO=2 BAR=3")] + m['include'] = [dict(python="2.7", env="FOO=2 BAR=3")] self.t.parse_envs() self.t.parse_matrix() self.assertEqual(self.t.matrix, [ - dict(python="python2.6", env=dict(FOO='1', BAR='2', CI='true')), - dict(python="python2.6", env=dict(FOO='2', BAR='1', CI='true')), - dict(python="python2.6", env=dict(FOO='2', BAR='3', CI='true')), + dict(python="2.7", env=dict(FOO='1', BAR='2', CI='true'), + os='linux', dist='precise', language='python'), + dict(python="2.7", env=dict(FOO='2', BAR='1', CI='true'), + os='linux', dist='precise', language='python'), + dict(python="2.7", env=dict(FOO='2', BAR='3', CI='true')), ]) diff --git a/buildbot_travis/travisyml.py b/buildbot_travis/travisyml.py index 44ed673..25d7e02 100644 --- a/buildbot_travis/travisyml.py +++ b/buildbot_travis/travisyml.py @@ -17,7 +17,9 @@ from __future__ import print_function from future.utils import string_types +import itertools import re +from copy import deepcopy import yaml from buildbot.plugins import util @@ -26,6 +28,18 @@ TRAVIS_HOOKS = ("before_install", "install", "after_install", "before_script", "script", "after_script") +DEFAULT_MATRIX = { + 'os': ( + 'linux', + ), + 'dist': ( + 'precise', + ), + 'language': { + 'python': ('2.7',), + }, +} + class TravisYmlInvalid(Exception): pass @@ -93,7 +107,7 @@ class TravisYml(object): Loads a .travis.yml file and parses it. """ - def __init__(self): + def __init__(self, cfgdict=None): self.language = None self.image = None self.environments = [{}] @@ -105,6 +119,10 @@ def __init__(self): self.email = TravisYmlEmail() self.irc = TravisYmlIrc() self.config = None + self.default_matrix = deepcopy(DEFAULT_MATRIX) + self.cfgdict = {} + if cfgdict: + self.cfgdict = cfgdict def parse(self, config_input): try: @@ -115,6 +133,7 @@ def parse(self, config_input): def parse_dict(self, config): self.config = config + self.load_cfgdict_options() self.parse_language() self.parse_label_mapping() self.parse_envs() @@ -124,6 +143,14 @@ def parse_dict(self, config): self.parse_notifications_email() self.parse_notifications_irc() + def load_cfgdict_options(self): + default_matrix = self.cfgdict.get('default_matrix') + if isinstance(default_matrix, dict): + self.default_matrix.update(default_matrix) + for k, v in self.default_matrix.iteritems(): + if isinstance(v, basestring): + self.default_matrix[k] = [v] + def parse_language(self): try: self.language = self.config['language'] @@ -184,17 +211,69 @@ def parse_branches(self): raise TravisYmlInvalid( "'branches' parameter contains neither 'only' nor 'except'") - def parse_matrix(self): + def _build_matrix(self): matrix = [] - python = self.config.get("python", ["python2.6"]) - if not isinstance(python, list): - python = [python] # First of all, build the implicit matrix - for lang in python: - for env in self.environments: - matrix.append(dict( - python=lang, - env=env, )) + supported_languages = self.default_matrix.get('language', {}) + language_options = supported_languages.get(self.language) + if not isinstance(language_options, (dict, tuple, list)): + language_options = [language_options] + # Many languages use their name as the key to check for versions to use. + if isinstance(language_options, (tuple, list)): + for language_version in self.config.get(self.language, language_options): + matrix.append({'language': self.language, + self.language: language_version}) + elif isinstance(language_options, dict): + # Get a view of the keys this language supports. Use those + # keys to check if they specified in the config, otherwise + # use the defaults. Do a cross-product across all of the + # keys to get all of the combinations. Finally, zip together + # the keys and the particular combination to convert to a + # dict to populate the matrix. + build_matrix_keys = sorted(list(language_options.keys())) + matrix_versions = [self.config.get(k, language_options[k]) + for k in build_matrix_keys] + # Ensure everything is at least a list of the versions for this + # language. + matrix_versions = [v if isinstance(v, (tuple, list)) else [v] + for v in matrix_versions] + for matrix_combination in itertools.product(*matrix_versions): + lang_matrix = dict(itertools.izip(build_matrix_keys, + matrix_combination)) + lang_matrix['language'] = self.language + matrix.append(lang_matrix) + + return matrix + + def _os_matrix(self, build_matrix): + # The language-level matrix has been built. Merge that with the os and + # dist options from the config. + matrix = [] + os_options = self.config.get('os', self.default_matrix['os']) + if isinstance(os_options, basestring): + os_options = [os_options] + dist_options = self.config.get('dist', self.default_matrix['dist']) + if isinstance(dist_options, basestring): + dist_options = [dist_options] + for os in os_options: + for dist in dist_options: + for build_config in build_matrix: + os_matrix = build_config.copy() + os_matrix['os'] = os + os_matrix['dist'] = dist + matrix.append(os_matrix) + + return matrix + + def parse_matrix(self): + build_matrix = self._build_matrix() + os_matrix = self._os_matrix(build_matrix) + matrix = [] + for env in self.environments: + for mat in os_matrix: + mat = mat.copy() + mat['env'] = env + matrix.append(mat) cfg = self.config.get("matrix", {})