From efdbee26ec5c800068e29fa4de493bc37855d0eb Mon Sep 17 00:00:00 2001 From: Nathan Muir Date: Sat, 15 Oct 2016 22:13:59 +1000 Subject: [PATCH 1/2] Move the core/configuration out of the CLI. Consolidate api/cli into a single core module. It's now much easier to load & call credsmash via python. Core - Perform version/compare before sealing new secret. DynamoDB - Create separate `get_one` and `get_latest`. This also removes the `name` & `version` from the ciphertext output, as it is an implementation detail of how DynamoDB stores data. --- HISTORY.md | 17 ++ credsmash/__init__.py | 6 +- credsmash/api/__init__.py | 8 - credsmash/api/delete.py | 20 -- credsmash/api/get.py | 11 - credsmash/api/list.py | 5 - credsmash/api/prune.py | 27 --- credsmash/cli.py | 249 +++----------------- credsmash/core.py | 318 ++++++++++++++++++++++++++ credsmash/dynamodb_storage_service.py | 26 +-- credsmash/templates.py | 18 +- 11 files changed, 385 insertions(+), 320 deletions(-) delete mode 100644 credsmash/api/__init__.py delete mode 100644 credsmash/api/delete.py delete mode 100644 credsmash/api/get.py delete mode 100644 credsmash/api/list.py delete mode 100644 credsmash/api/prune.py create mode 100644 credsmash/core.py diff --git a/HISTORY.md b/HISTORY.md index ea774ae..858b764 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -140,3 +140,20 @@ - By default `credsmash put` will check if the value of a secret has changed. Use `--version` or `--no-compare` to avoid this comparison. + + - `credsmash` has a straight-forward API for pythonic access. + + ```py + import credsmash + # Auto-configure a session from your + # /etc/credsmash.cfg or CREDSMASH_CONFIG environment variable + session = credsmash.get_session() + # or, provide options directly- + session = credsmash.get_session(table_name='my-dynamodb-table', key_id='my-key') + + # Access a secret + plaintext = session.get_one('my_secret') + # search for secrets + secret_name, plaintext = session.find_one('s3_prod_*') + secrets = session.find_many('s3_*') + ``` diff --git a/credsmash/__init__.py b/credsmash/__init__.py index 00c0723..e7d18f0 100755 --- a/credsmash/__init__.py +++ b/credsmash/__init__.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # Copyright 2015 Luminal, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,5 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pkg_resources -__version__ = pkg_resources.resource_string(__name__, 'VERSION') +__version__ = __import__('pkg_resources').resource_string(__name__, 'VERSION') + +from .core import get_session, Credsmash diff --git a/credsmash/api/__init__.py b/credsmash/api/__init__.py deleted file mode 100644 index c1e93f1..0000000 --- a/credsmash/api/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -from __future__ import absolute_import, division, print_function, unicode_literals - - -from .delete import delete_secret -from .get import get_secret -from .list import list_secrets -from .prune import prune_secret -from .put import put_secret diff --git a/credsmash/api/delete.py b/credsmash/api/delete.py deleted file mode 100644 index f4c0db2..0000000 --- a/credsmash/api/delete.py +++ /dev/null @@ -1,20 +0,0 @@ -from __future__ import absolute_import, division, print_function, unicode_literals - -import logging - -logger = logging.getLogger(__name__) - - -def delete_secret(storage_service, secret_name): - secrets = storage_service.list_one(secret_name) - if not secrets: - logger.info('Not found: %s', secret_name) - return - - for secret in secrets: - logger.info("Deleting %s -- version %s", - secret["name"], secret["version"]) - storage_service.delete_one( - secret["name"], secret["version"] - ) - logger.info('Deleted %s', secret_name) diff --git a/credsmash/api/get.py b/credsmash/api/get.py deleted file mode 100644 index 9795021..0000000 --- a/credsmash/api/get.py +++ /dev/null @@ -1,11 +0,0 @@ -from __future__ import absolute_import, division, print_function, unicode_literals - -from credsmash.crypto import open_secret -from credsmash.util import ItemNotFound - - -def get_secret(storage_service, key_service, secret_name, version=None): - ciphertext = storage_service.get_one(secret_name, version=version) - if not ciphertext: - raise ItemNotFound("Item {'name': '%s', 'version': %s} couldn't be found." % (secret_name, version)) - return open_secret(key_service, ciphertext) diff --git a/credsmash/api/list.py b/credsmash/api/list.py deleted file mode 100644 index 1d7f500..0000000 --- a/credsmash/api/list.py +++ /dev/null @@ -1,5 +0,0 @@ -from __future__ import absolute_import, division, print_function, unicode_literals - - -def list_secrets(storage_service): - return storage_service.list_all() diff --git a/credsmash/api/prune.py b/credsmash/api/prune.py deleted file mode 100644 index db838ba..0000000 --- a/credsmash/api/prune.py +++ /dev/null @@ -1,27 +0,0 @@ -from __future__ import absolute_import, division, print_function, unicode_literals - -import logging - -logger = logging.getLogger(__name__) - - -def prune_secret(storage_service, secret_name): - secrets = storage_service.list_one(secret_name) - if not secrets: - logger.info('Not found: %s', secret_name) - return - - max_version = max( - secret['version'] - for secret in secrets - ) - - for secret in secrets: - if secret['version'] == max_version: - continue - logger.info("Deleting %s -- version %s", - secret["name"], secret["version"]) - storage_service.delete_one( - secret['name'], secret['version'] - ) - logger.info('Pruned %s (current version=%d)', secret_name, max_version) diff --git a/credsmash/cli.py b/credsmash/cli.py index 1fb58a6..6751bde 100644 --- a/credsmash/cli.py +++ b/credsmash/cli.py @@ -1,10 +1,5 @@ from __future__ import absolute_import, division, print_function -import codecs -import fnmatch -import logging -import operator -import os import sys import boto3 @@ -12,74 +7,15 @@ import pkg_resources import six -import credsmash.api -from credsmash.crypto import ALGO_AES_CTR +from credsmash import get_session from credsmash.util import set_stream_logger, detect_format, \ - parse_config, read_one, read_many, write_one, write_many - -logger = logging.getLogger(__name__) - - -class Environment(object): - def __init__(self, storage_service_name, storage_service_config, - key_service_name, key_service_config, - algorithm, algorithm_options): - self._storage_service = None - self.storage_service_name = storage_service_name - self.storage_service_config = storage_service_config - self._key_service = None - self.key_service_name = key_service_name - self.key_service_config = key_service_config - self.algorithm = algorithm - self.algorithm_options = algorithm_options - self._session = None - - @property - def session(self): - if self._session is None: - self._session = boto3.Session() - return self._session - - @staticmethod - def load_entry_point(group, name): - entry_points = pkg_resources.iter_entry_points( - group, name - ) - for entry_point in entry_points: - return entry_point.load() - raise RuntimeError('Not found EntryPoint(group={0},name={1})'.format(group, name)) - - @property - def key_service(self): - if not self._key_service: - cls = self.load_entry_point('credsmash.key_service', self.key_service_name) - self._key_service = cls( - session=self.session, - **self.key_service_config - ) - logger.debug('key_service=%r', self._key_service) - return self._key_service - - @property - def storage_service(self): - if not self._storage_service: - cls = self.load_entry_point('credsmash.storage_service', self.storage_service_name) - self._storage_service = cls( - session=self.session, - **self.storage_service_config - ) - logger.debug('storage_service=%r', self._storage_service) - return self._storage_service - - def __repr__(self): - return 'Environment(storage_service=%r,key_service=%r)' % (self._storage_service, self._key_service) + read_one, read_many, write_one, write_many @click.group() @click.option('--config', '-c', - envvar='CREDSMASH_CONFIG', type=click.Path(resolve_path=True), - default='/etc/credsmash.cfg') + default=None) @click.option('--table-name', '-t', default=None, help="DynamoDB table to use for " "credential storage") @@ -92,59 +28,15 @@ def __repr__(self): "Only works if --key-id is passed.") @click.pass_context def main(ctx, config, table_name, key_id, context=None): - config_data = {} - sections = {} - if os.path.exists(config): - with codecs.open(config, 'r') as config_fp: - sections = parse_config(config_fp) - config_data = sections.get('credsmash', {}) - - if key_id: - # Using --key-id/-k will ignore the configuration file. - key_service_name = 'kms' - key_service_config = { - 'key_id': key_id - } - if context: - key_service_config['encryption_context'] = dict(context) - else: - if context: - logger.warning('--context can only be used in conjunction with --key-id') - key_service_name = config_data.get('key_service', 'kms') - key_service_config = sections.get('credsmash:key_service:%s' % key_service_name, {}) - if key_service_name == 'kms': - key_service_config.setdefault( - 'key_id', config_data.get('key_id', 'alias/credsmash') - ) - - if table_name: - storage_service_name = 'dynamodb' - storage_service_config = { - 'table_name': table_name - } - else: - storage_service_name = config_data.get('storage_service', 'dynamodb') - storage_service_config = sections.get('credsmash:storage_service:%s' % storage_service_name, {}) - if storage_service_name == 'dynamodb': - storage_service_config.setdefault( - 'table_name', config_data.get('table_name', 'secret-store') - ) - - algorithm = config_data.get('algorithm', ALGO_AES_CTR) - algorithm_options = sections.get('credsmash:%s' % algorithm, {}) - - set_stream_logger( - level=config_data.get('log_level', 'INFO') + ctx.obj = get_session( + config=config, + table_name=table_name, + key_id=key_id, + context=context ) - env = Environment( - storage_service_name, - storage_service_config, - key_service_name, - key_service_config, - algorithm, - algorithm_options + set_stream_logger( + level=ctx.obj.log_level ) - ctx.obj = env @main.command('list') @@ -154,25 +46,18 @@ def cmd_list(ctx, pattern=None): """ List all secrets & their versions. """ - secrets = credsmash.api.list_secrets( - ctx.obj.storage_service - ) if pattern: - matched_names = set(fnmatch.filter((secret['name'] for secret in secrets), pattern)) - secrets = [ - secret - for secret in secrets - if secret['name'] in matched_names - ] + secrets = ctx.obj.list_filtered(pattern) + else: + secrets = ctx.obj.list_all() if not secrets: return - max_len = max(len(secret["name"]) for secret in secrets) - for cred in sorted(secrets, key=operator.itemgetter("name", "version")): - click.echo( - "{cred[name]:{l}} -- version {cred[version]:>}".format(l=max_len, cred=cred) - ) + max_len = max(len(secret_name) for secret_name, _ in secrets) + for secret_name, version in sorted(secrets): + click.echo("{0:{l}} -- version {1:>}".format( + secret_name, version, l=max_len)) @main.command('delete') @@ -180,11 +65,9 @@ def cmd_list(ctx, pattern=None): @click.pass_context def cmd_delete_one(ctx, secret_name): """ - Delete every version of a single secret + Delete every record of a secret """ - credsmash.api.delete_secret( - ctx.obj.storage_service, secret_name - ) + ctx.obj.delete_one(secret_name) @main.command('delete-many') @@ -192,16 +75,9 @@ def cmd_delete_one(ctx, secret_name): @click.pass_context def cmd_delete_many(ctx, pattern): """ - Delete every version of all matching secrets + Delete every record of all matching secrets """ - secret_names = { - x["name"] for x in credsmash.api.list_secrets(ctx.obj.storage_service) - } - secret_names = fnmatch.filter(secret_names, pattern) - for secret_name in secret_names: - credsmash.api.delete_secret( - ctx.obj.storage_service, secret_name - ) + ctx.obj.delete_many(pattern) @main.command('prune') @@ -211,9 +87,7 @@ def cmd_prune_one(ctx, secret_name): """ Delete all but the latest version of a single secret """ - credsmash.api.prune_secret( - ctx.obj.storage_service, secret_name - ) + ctx.obj.prune_one(secret_name) @main.command('prune-many') @@ -223,14 +97,7 @@ def cmd_prune_many(ctx, pattern): """ Delete all but the latest version of all matching secrets """ - secret_names = { - x["name"] for x in credsmash.api.list_secrets(ctx.obj.storage_service) - } - secret_names = fnmatch.filter(secret_names, pattern) - for secret_name in secret_names: - credsmash.api.prune_secret( - ctx.obj.storage_service, secret_name - ) + ctx.obj.prune_many(pattern) @main.command('get') @@ -243,9 +110,7 @@ def cmd_get_one(ctx, secret_name, destination, fmt=None, version=None): """ Fetch the latest, or a specific version of a secret """ - secret_value = credsmash.api.get_secret( - ctx.obj.storage_service, - ctx.obj.key_service, + secret_value = ctx.obj.get_one( secret_name, version=version, ) @@ -262,17 +127,7 @@ def cmd_get_all(ctx, destination, fmt): """ Fetch the latest version of all secrets """ - secret_names = { - x["name"] for x in credsmash.api.list_secrets(ctx.obj.storage_service) - } - secrets = { - secret_name: credsmash.api.get_secret( - ctx.obj.storage_service, - ctx.obj.key_service, - secret_name, - ) - for secret_name in secret_names - } + secrets = ctx.obj.get_all() if not fmt: fmt = detect_format(destination, default_format='json') write_many(secrets, destination, fmt) @@ -288,22 +143,7 @@ def cmd_find_one(ctx, pattern, destination, fmt=None, version=None): """ Find exactly one secret matching """ - secret_names = { - x["name"] for x in credsmash.api.list_secrets(ctx.obj.storage_service) - } - secret_names = fnmatch.filter(secret_names, pattern) - if not secret_names: - raise click.ClickException('No matching secrets found for pattern={0}'.format(pattern)) - if len(secret_names) > 1: - raise click.ClickException('Too many results ({0}) for pattern={1}'.format(len(secret_names), pattern)) - - secret_name = secret_names[0] - secret_value = credsmash.api.get_secret( - ctx.obj.storage_service, - ctx.obj.key_service, - secret_name, - version=version, - ) + secret_name, secret_value = ctx.obj.find_one(pattern) if not fmt: fmt = detect_format(destination, default_format='raw') write_one(secret_name, secret_value, destination, fmt) @@ -318,18 +158,7 @@ def cmd_find_many(ctx, pattern, destination, fmt=None): """ Find all secrets matching """ - secret_names = { - x["name"] for x in credsmash.api.list_secrets(ctx.obj.storage_service) - } - secret_names = fnmatch.filter(secret_names, pattern) - secrets = { - secret_name: credsmash.api.get_secret( - ctx.obj.storage_service, - ctx.obj.key_service, - secret_name, - ) - for secret_name in secret_names - } + secrets = ctx.obj.find_many(pattern) if not fmt: fmt = detect_format(destination, default_format='json') write_many(secrets, destination, fmt) @@ -351,18 +180,11 @@ def cmd_put_one(ctx, secret_name, source, fmt=None, version=None, compare=True): fmt = detect_format(source, default_format='raw') secret_value = read_one(secret_name, source, fmt) - stored_version = credsmash.api.put_secret( - ctx.obj.storage_service, - ctx.obj.key_service, + ctx.obj.put_one( secret_name, secret_value, version=version, compare=compare, - algorithm=ctx.obj.algorithm, - **ctx.obj.algorithm_options - ) - logger.info( - 'Stored {0} @ version {1}'.format(secret_name, stored_version) ) @@ -379,20 +201,7 @@ def cmd_put_many(ctx, source, fmt, compare=True): if not fmt: fmt = detect_format(source, default_format='json') secrets = read_many(source, fmt) - - for secret_name, secret_value in secrets.items(): - stored_version = credsmash.api.put_secret( - ctx.obj.storage_service, - ctx.obj.key_service, - secret_name, - secret_value, - version=None, - compare=compare, - algorithm=ctx.obj.algorithm, - **ctx.obj.algorithm_options - ) - logger.info('Stored {0} @ version {1}'.format(secret_name, stored_version)) - logger.debug('Stored {0} secrets'.format(len(secrets))) + ctx.obj.put_many(secrets, compare=compare) # Load any extra CLI's diff --git a/credsmash/core.py b/credsmash/core.py new file mode 100644 index 0000000..fc9fcda --- /dev/null +++ b/credsmash/core.py @@ -0,0 +1,318 @@ +from __future__ import absolute_import, division, print_function, unicode_literals + +import codecs +import fnmatch +import logging +import os + +import pkg_resources + +from .crypto import ALGO_AES_CTR, open_secret, seal_secret +from .util import parse_config, ItemNotFound + +logger = logging.getLogger(__name__) + + +class Credsmash(object): + def __init__(self, storage_service_loader, key_service_loader, + algorithm, algorithm_options, log_level): + self.storage_service_loader = storage_service_loader + self.key_service_loader = key_service_loader + self.algorithm = algorithm + self.algorithm_options = algorithm_options + self.log_level = log_level + + @property + def key_service(self): + return self.key_service_loader.get() + + @property + def storage_service(self): + return self.storage_service_loader.get() + + def list_all(self): + """ + List all secrets & their versions + + :rtype: List[Tuple[text, int]] + """ + return self.storage_service.list_all() + + def _list_all_names(self): + """ + :rtype: Set[text] + """ + return { + secret_name for secret_name, _ in self.list_all() + } + + def list_filtered(self, pattern): + """ + List all secrets & their versions matching + + :type pattern: text + :rtype: List[Tuple[text, int]] + """ + return [ + (secret_name, version) + for secret_name, version in self.list_all() + if fnmatch.fnmatch(secret_name, pattern) + ] + + def _list_filtered_names(self, pattern): + """ + :rtype: Set[unicode] + """ + return { + secret_name for secret_name, _ in self.list_filtered(pattern) + } + + def get_one(self, secret_name, version=None): + """ + Get a secret... + :type secret_name: text + :type version: int + :rtype: bytes + """ + if version is None: + version, ciphertext = self.storage_service.get_latest(secret_name) + else: + ciphertext = self.storage_service.get_one(secret_name, version) + if not ciphertext: + raise ItemNotFound( + "Item name={0} version={1} couldn't be found.".format(secret_name, version) + ) + return open_secret(self.key_service, ciphertext) + + def get_all(self): + """ + Get the latest version of all secrets + :rtype: Dict[text, bytes] + """ + return { + secret_name: self.get_one(secret_name) + for secret_name in self._list_all_names() + } + + def find_one(self, pattern): + """ + Find exactly one secret matching + :type pattern: text + :rtype: Tuple[text, bytes] + """ + secret_names = self._list_filtered_names(pattern) + + # TODO - Audit/Evaluate the type of errors thrown here + if not secret_names: + raise ItemNotFound("Not found pattern={0}".format(pattern)) + if len(secret_names) > 1: + raise RuntimeError( + 'Found {0} secrets matching pattern={1} matches={2}'.format( + len(secret_names), pattern, ",".join(secret_names) + ) + ) + + secret_name = secret_names.pop() + plaintext = self.get_one(secret_name) + return secret_name, plaintext + + def find_many(self, pattern): + """ + Find all secrets matching + + :type pattern: text + :rtype: Dict[text, bytes] + """ + return { + secret_name: self.get_one(secret_name) + for secret_name in self._list_filtered_names(pattern) + } + + def put_one(self, secret_name, plaintext, version=None, compare=True): + """ + Store a secret + + :type secret_name: text + :type plaintext: bytes + :type version: Optional[int] + :type compare: bool + :rtype: int + :return: The latest version of the secret + """ + if version is None: + latest_version, latest_ciphertext = self.storage_service.get_latest(secret_name) + version = 1 + if latest_ciphertext: + version += latest_version + if compare: + latest_plaintext = open_secret(self.key_service, latest_ciphertext) + if plaintext == latest_plaintext: + logger.info('"%s" is unchanged from version %d', secret_name, latest_version) + return latest_version + + sealed = seal_secret( + self.key_service, + plaintext, + algorithm=self.algorithm, + binary_type=getattr(self.storage_service, 'binary_type', None), + **self.algorithm_options + ) + + self.storage_service.put_one(secret_name, version, sealed) + logger.info('Stored %s @ version %d', secret_name, version) + return version + + def put_many(self, secrets, compare=True): + """ + Store many secrets + + :type secrets: Dict[text, bytes] + :type compare: bool + """ + for secret_name, plaintext in secrets.items(): + self.put_one( + secret_name, + plaintext, + version=None, + compare=compare, + ) + logger.debug('Stored %d secrets', len(secrets)) + + def delete_one(self, secret_name): + """ + Delete every record of a secret + :type secret_name: text + """ + secrets = self.storage_service.list_one(secret_name) + if not secrets: + logger.info('Not found: %s', secret_name) + return + + for secret_name, version in secrets: + logger.info("Deleting %s -- version %s", + secret_name, version) + self.storage_service.delete_one( + secret_name, version + ) + logger.info('Deleted %s', secret_name) + + def delete_many(self, pattern): + """ + Delete every record of all matching secrets + :type pattern: text + """ + for secret_name in self._list_filtered_names(pattern): + self.delete_one(secret_name) + + def prune_one(self, secret_name): + """ + Delete all but the latest version of a single secret + :type secret_name: text + """ + secrets = self.storage_service.list_one(secret_name) + if not secrets: + logger.info('Not found: %s', secret_name) + return + + max_version = max( + version + for _, version in secrets + ) + + for secret_name, version in secrets: + if version == max_version: + continue + logger.info("Deleting %s -- version %s", + secret_name, version) + self.storage_service.delete_one( + secret_name, version + ) + logger.info('Pruned %s (current version=%d)', secret_name, max_version) + + def prune_many(self, pattern): + """ + Delete all but the latest version of all matching secrets + :type pattern: text + """ + for secret_name in self._list_filtered_names(pattern): + self.prune_one(secret_name) + + +def get_session(config=None, table_name=None, key_id=None, context=None): + if config is None: + config = os.environ.get('CREDSMASH_CONFIG', '/etc/credsmash.cfg') + + """Creates a new credsmash session.""" + main_section = {} + sections = {} + if config and os.path.exists(config): + with codecs.open(config, 'r') as config_fp: + sections = parse_config(config_fp) + main_section = sections.get('credsmash', {}) + + if key_id: + # Using --key-id/-k will ignore the configuration file. + key_service_name = 'kms' + key_service_config = { + 'key_id': key_id + } + if context: + key_service_config['encryption_context'] = dict(context) + else: + if context: + logger.warning('--context can only be used in conjunction with --key-id') + key_service_name = main_section.get('key_service', 'kms') + key_service_config = sections.get('credsmash:key_service:%s' % key_service_name, {}) + if key_service_name == 'kms': + key_service_config.setdefault( + 'key_id', main_section.get('key_id', 'alias/credsmash') + ) + + if table_name: + storage_service_name = 'dynamodb' + storage_service_config = { + 'table_name': table_name + } + else: + storage_service_name = main_section.get('storage_service', 'dynamodb') + storage_service_config = sections.get('credsmash:storage_service:%s' % storage_service_name, {}) + if storage_service_name == 'dynamodb': + storage_service_config.setdefault( + 'table_name', main_section.get('table_name', 'secret-store') + ) + + algorithm = main_section.get('algorithm', ALGO_AES_CTR) + algorithm_options = sections.get('credsmash:%s' % algorithm, {}) + + return Credsmash( + EntryPointLoader('credsmash.storage_service', storage_service_name, **storage_service_config), + EntryPointLoader('credsmash.key_service', key_service_name, **key_service_config), + algorithm, + algorithm_options, + log_level=main_section.get('log_level', 'INFO') + ) + + +class EntryPointLoader(object): + def __init__(self, group, name, *args, **kwargs): + self.group = group + self.name = name + self.args = args + self.kwargs = kwargs + self._obj = None + + def get(self): + if not self._obj: + cls = self.load_entry_point(self.group, self.name) + self._obj = cls(*self.args, **self.kwargs) + self.args, self.kwargs = None, None + return self._obj + + @staticmethod + def load_entry_point(group, name): + entry_points = pkg_resources.iter_entry_points( + group, name + ) + for entry_point in entry_points: + return entry_point.load() + raise RuntimeError('Not found EntryPoint(group={0},name={1})'.format(group, name)) diff --git a/credsmash/dynamodb_storage_service.py b/credsmash/dynamodb_storage_service.py index fa59001..4ab8d60 100644 --- a/credsmash/dynamodb_storage_service.py +++ b/credsmash/dynamodb_storage_service.py @@ -58,7 +58,7 @@ def list_one(self, name): ExpressionAttributeNames={"#N": "name"} ) return [ - self._unwrap_doc(item) + (item['name'], self._unwrap_int(item['version']),) for item in items ] @@ -68,7 +68,7 @@ def list_all(self): ExpressionAttributeNames={"#N": "name"} ) return [ - self._unwrap_doc(item) + (item['name'], self._unwrap_int(item['version']),) for item in items ] @@ -89,13 +89,7 @@ def put_one(self, name, version, ciphertext): ConditionExpression=Attr('name').not_exists() ) - def get_one(self, name, version=None): - if not version: - return self._get_latest_secret(name) - else: - return self._get_versioned_secret(name, version) - - def _get_latest_secret(self, secret_name): + def get_latest(self, secret_name): # do a consistent fetch of the credential with the highest version response = self.secrets_table.query( Limit=1, @@ -104,15 +98,21 @@ def _get_latest_secret(self, secret_name): KeyConditionExpression=ConditionKey("name").eq(secret_name) ) if response["Count"] == 0: - return - return self._unwrap_doc(response["Items"][0]) + return None, None + ciphertext = self._unwrap_doc(response["Items"][0]) + assert ciphertext.pop('name') == secret_name + version = ciphertext.pop('version') + return version, ciphertext - def _get_versioned_secret(self, secret_name, version): + def get_one(self, secret_name, version): version = self._wrap_int(version) response = self.secrets_table.get_item(Key={"name": secret_name, "version": version}) if "Item" not in response: return - return self._unwrap_doc(response["Item"]) + ciphertext = self._unwrap_doc(response["Item"]) + assert ciphertext.pop('name') == secret_name + assert ciphertext.pop('version') == version + return ciphertext @classmethod def _unwrap_doc(cls, obj): diff --git a/credsmash/templates.py b/credsmash/templates.py index ad32457..bdd0248 100644 --- a/credsmash/templates.py +++ b/credsmash/templates.py @@ -11,7 +11,6 @@ import click import jinja2.sandbox -import credsmash.api from .util import read_many, read_many_str, minjson, envfile_quote, shell_quote, parse_manifest, detect_format, ItemNotFound from .cli import main @@ -19,9 +18,8 @@ class CredsmashProxy(object): - def __init__(self, key_service, storage_service, key_fmt, encoding='utf-8', errors='strict'): - self._key_service = key_service - self._storage_service = storage_service + def __init__(self, session, key_fmt, encoding='utf-8', errors='strict'): + self.session = session self._key_fmt = key_fmt self._data = {} self._encoding = encoding @@ -47,11 +45,7 @@ def get_bytes(self, key): lookup_key = self._key_fmt.format(key) logger.debug('key=%s lookup_key=%s', key, lookup_key) try: - res = credsmash.api.get_secret( - self._storage_service, - self._key_service, - key, - ) + res = self.session.get_one(key) except ItemNotFound: raise KeyError(repr(key)) self._data[key] = res @@ -128,8 +122,7 @@ def cmd_render_template( secrets = DictProxy(local_secrets, key_fmt) else: secrets = CredsmashProxy( - ctx.obj.key_service, - ctx.obj.storage_service, + ctx.obj, key_fmt, ) @@ -178,8 +171,7 @@ def cmd_render_template( secrets = DictProxy(local_secrets, key_fmt) else: secrets = CredsmashProxy( - ctx.obj.key_service, - ctx.obj.storage_service, + ctx.obj, key_fmt, ) From b232908c3dbd04d79ac27096a840479b7dba6261 Mon Sep 17 00:00:00 2001 From: Nathan Muir Date: Thu, 23 Mar 2017 12:37:31 +1000 Subject: [PATCH 2/2] Update KMS to support Additional Authenticated Data Crypto - Support additional authenticated data on key service --- credsmash/core.py | 13 +++++++-- credsmash/crypto/__init__.py | 53 ++++++++++++++++++++++++++++-------- credsmash/crypto/aes_ctr.py | 32 ++++++++++++---------- credsmash/crypto/aes_gcm.py | 17 ++++++------ credsmash/kms_key_service.py | 18 +++++++++--- tests/test_crypto.py | 34 ++++++++++++++--------- 6 files changed, 114 insertions(+), 53 deletions(-) diff --git a/credsmash/core.py b/credsmash/core.py index fc9fcda..17ebc3d 100644 --- a/credsmash/core.py +++ b/credsmash/core.py @@ -82,7 +82,9 @@ def get_one(self, secret_name, version=None): raise ItemNotFound( "Item name={0} version={1} couldn't be found.".format(secret_name, version) ) - return open_secret(self.key_service, ciphertext) + return open_secret(self.key_service, ciphertext, metadata={ + 'name': secret_name, 'version': version, + }) def get_all(self): """ @@ -145,7 +147,11 @@ def put_one(self, secret_name, plaintext, version=None, compare=True): if latest_ciphertext: version += latest_version if compare: - latest_plaintext = open_secret(self.key_service, latest_ciphertext) + latest_plaintext = open_secret( + self.key_service, latest_ciphertext, metadata={ + 'name': secret_name, 'version': latest_version, + } + ) if plaintext == latest_plaintext: logger.info('"%s" is unchanged from version %d', secret_name, latest_version) return latest_version @@ -155,6 +161,9 @@ def put_one(self, secret_name, plaintext, version=None, compare=True): plaintext, algorithm=self.algorithm, binary_type=getattr(self.storage_service, 'binary_type', None), + metadata={ + 'name': secret_name, 'version': version + }, **self.algorithm_options ) diff --git a/credsmash/crypto/__init__.py b/credsmash/crypto/__init__.py index 46f2ec2..cdb6d9f 100644 --- a/credsmash/crypto/__init__.py +++ b/credsmash/crypto/__init__.py @@ -7,47 +7,78 @@ from .aes_gcm import open_aes_gcm, seal_aes_gcm, ALGO_AES_GCM -def seal_secret(key_service, plaintext, algorithm=ALGO_AES_CTR, binary_type=None, **seal_kwargs): +def seal_secret(key_service, plaintext, metadata, + algorithm=ALGO_AES_CTR, protocol=2, binary_type=None, **seal_kwargs): if isinstance(plaintext, six.text_type): plaintext = plaintext.encode('utf-8') + if protocol == 1: + additional_authenticated_data = {} + elif protocol == 2: + additional_authenticated_data = { + 'name': metadata['name'], + 'version': metadata['version'], + } + else: + raise RuntimeError('Unsupported protocol: %s' % protocol) + if not algorithm: algorithm = ALGO_AES_CTR if algorithm == ALGO_AES_GCM: - return seal_aes_gcm( + ciphertext = seal_aes_gcm( key_service, plaintext, + additional_authenticated_data, binary_type=binary_type, **seal_kwargs ) - if algorithm == ALGO_AES_CTR: - return seal_aes_ctr( + elif algorithm == ALGO_AES_CTR: + ciphertext = seal_aes_ctr( key_service, plaintext, + additional_authenticated_data, binary_type=binary_type, **seal_kwargs ) - if algorithm == ALGO_AES_CTR_LEGACY: - return seal_aes_ctr_legacy( + elif algorithm == ALGO_AES_CTR_LEGACY: + ciphertext = seal_aes_ctr_legacy( key_service, plaintext, + additional_authenticated_data, **seal_kwargs ) - raise RuntimeError('Unsupported algo: %s' % algorithm) + else: + raise RuntimeError('Unsupported algo: %s' % algorithm) + + ciphertext.update({ + 'protocol': protocol, + }) + return ciphertext + +def open_secret(key_service, ciphertext, metadata): + protocol = ciphertext.get('protocol', 1) + if protocol == 1: + additional_authenticated_data = {} + elif protocol == 2: + additional_authenticated_data = { + 'name': metadata['name'], + 'version': metadata['version'], + } + else: + raise RuntimeError('Unsupported protocol: %s' % protocol) -def open_secret(key_service, ciphertext): algorithm = ciphertext.get('algorithm') if algorithm == ALGO_AES_GCM: - return open_aes_gcm(key_service, ciphertext) + return open_aes_gcm(key_service, ciphertext, additional_authenticated_data) if algorithm == ALGO_AES_CTR: - return open_aes_ctr(key_service, ciphertext) + return open_aes_ctr(key_service, ciphertext, additional_authenticated_data) if not algorithm or algorithm == ALGO_AES_CTR_LEGACY: - return open_aes_ctr_legacy(key_service, ciphertext) + return open_aes_ctr_legacy(key_service, ciphertext, additional_authenticated_data) raise RuntimeError('Unsupported algo: %s' % algorithm) diff --git a/credsmash/crypto/aes_ctr.py b/credsmash/crypto/aes_ctr.py index 3ecebf3..3a5a55b 100644 --- a/credsmash/crypto/aes_ctr.py +++ b/credsmash/crypto/aes_ctr.py @@ -27,30 +27,31 @@ LEGACY_NONCE = b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01' -def open_aes_ctr(key_service, material): +def open_aes_ctr(key_service, ciphertext, additional_authenticated_data): """ Decrypts secrets stored by `seal_aes_ctr`. Allows for binary plaintext """ - key = key_service.decrypt(material['key']) - digest_method = material.get('digest', DEFAULT_DIGEST) - ciphertext = material['contents'] - hmac = material['hmac'] - nonce = material.get('nonce', LEGACY_NONCE) - return _open_aes_ctr(key, nonce, ciphertext, hmac, digest_method) + key = key_service.decrypt(ciphertext['key'], additional_authenticated_data) + digest_method = ciphertext.get('digest', DEFAULT_DIGEST) + contents = ciphertext['contents'] + hmac = ciphertext['hmac'] + nonce = ciphertext.get('nonce', LEGACY_NONCE) + return _open_aes_ctr(key, nonce, contents, hmac, digest_method) -def seal_aes_ctr(key_service, secret, digest_method=DEFAULT_DIGEST, key_length=DEFAULT_KEY_LENGTH, binary_type=None): +def seal_aes_ctr(key_service, plaintext, additional_authenticated_data, + digest_method=DEFAULT_DIGEST, key_length=DEFAULT_KEY_LENGTH, binary_type=None): """ Encrypts `secret` using the key service. You can decrypt with the companion method `open_aes_ctr`. """ - key, encoded_key = key_service.generate_key_data(key_length) + key, encoded_key = key_service.generate_key_data(key_length, additional_authenticated_data) nonce = os.urandom(16) ciphertext, hmac = _seal_aes_ctr( - secret, key, nonce, digest_method + plaintext, key, nonce, digest_method ) # Agh! a mighty break in abstraction @@ -70,20 +71,21 @@ def Binary(value): } -def open_aes_ctr_legacy(key_service, material): +def open_aes_ctr_legacy(key_service, ciphertext, additional_authenticated_data): """ Decrypts secrets stored by `seal_aes_ctr_legacy`. Assumes that the plaintext is str (non-binary). """ - key = key_service.decrypt(_from_b64(material['key'])) + key = key_service.decrypt(_from_b64(material['key']), additional_authenticated_data) digest_method = material.get('digest', DEFAULT_DIGEST) ciphertext = _from_b64(material['contents']) hmac = _from_hex(material['hmac']) return _open_aes_ctr(key, LEGACY_NONCE, ciphertext, hmac, digest_method) -def seal_aes_ctr_legacy(key_service, secret, digest_method=DEFAULT_DIGEST): +def seal_aes_ctr_legacy(key_service, plaintext, additional_authenticated_data, + digest_method=DEFAULT_DIGEST): """ :deprecated: Please use `seal_aes_ctr` instead. @@ -93,9 +95,9 @@ def seal_aes_ctr_legacy(key_service, secret, digest_method=DEFAULT_DIGEST): """ # generate a a 64 byte key. # Half will be for data encryption, the other half for HMAC - key, encoded_key = key_service.generate_key_data(64) + key, encoded_key = key_service.generate_key_data(64, additional_authenticated_data) ciphertext, hmac = _seal_aes_ctr( - secret, key, LEGACY_NONCE, digest_method, + plaintext, key, LEGACY_NONCE, digest_method, ) return { 'key': _to_b64(encoded_key), diff --git a/credsmash/crypto/aes_gcm.py b/credsmash/crypto/aes_gcm.py index 1db9b25..17315e7 100644 --- a/credsmash/crypto/aes_gcm.py +++ b/credsmash/crypto/aes_gcm.py @@ -10,29 +10,30 @@ DEFAULT_IV_LENGTH = 12 -def open_aes_gcm(key_service, material): +def open_aes_gcm(key_service, ciphertext, additional_authenticated_data): """ Decrypts secrets stored by `seal_aes_ctr`. Allows for binary plaintext """ - key = key_service.decrypt(material['key']) + key = key_service.decrypt(ciphertext['key'], additional_authenticated_data) return _open_aes_gcm( key, - material['iv'], - material['tag'], - material['contents'] + ciphertext['iv'], + ciphertext['tag'], + ciphertext['contents'] ) -def seal_aes_gcm(key_service, secret, key_length=DEFAULT_KEY_LENGTH, iv_length=DEFAULT_IV_LENGTH, binary_type=None): +def seal_aes_gcm(key_service, plaintext, additional_authenticated_data, + key_length=DEFAULT_KEY_LENGTH, iv_length=DEFAULT_IV_LENGTH, binary_type=None): """ Encrypts `secret` using the key service. You can decrypt with the companion method `open_aes_ctr`. """ - key, encoded_key = key_service.generate_key_data(key_length) - iv, tag, ciphertext = _seal_aes_gcm(secret, key, iv_length) + key, encoded_key = key_service.generate_key_data(key_length, additional_authenticated_data) + iv, tag, ciphertext = _seal_aes_gcm(plaintext, key, iv_length) # Agh! a mighty break in abstraction # DynamoDB wont put `bytes` => `Binary` in python2 diff --git a/credsmash/kms_key_service.py b/credsmash/kms_key_service.py index 93c9509..d30adcb 100644 --- a/credsmash/kms_key_service.py +++ b/credsmash/kms_key_service.py @@ -11,20 +11,22 @@ def __init__(self, session, key_id, encryption_context=None): encryption_context = {} self.encryption_context = encryption_context - def generate_key_data(self, number_of_bytes): + def generate_key_data(self, number_of_bytes, additional_authenticated_data=None): + encryption_context = self._get_encryption_context(additional_authenticated_data) try: kms_response = self.kms.generate_data_key( - KeyId=self.key_id, EncryptionContext=self.encryption_context, NumberOfBytes=int(number_of_bytes) + KeyId=self.key_id, EncryptionContext=encryption_context, NumberOfBytes=int(number_of_bytes) ) except: raise KmsError("Could not generate key using KMS key %s" % self.key_id) return kms_response['Plaintext'], kms_response['CiphertextBlob'] - def decrypt(self, encoded_key): + def decrypt(self, encoded_key, additional_authenticated_data=None): + encryption_context = self._get_encryption_context(additional_authenticated_data) try: kms_response = self.kms.decrypt( CiphertextBlob=encoded_key, - EncryptionContext=self.encryption_context + EncryptionContext=encryption_context ) except botocore.exceptions.ClientError as e: if e.response["Error"]["Code"] == "InvalidCiphertextException": @@ -41,6 +43,14 @@ def decrypt(self, encoded_key): raise KmsError(msg) return kms_response['Plaintext'] + def _get_encryption_context(self, additional_authenticated_data): + encryption_context = {} + if self.encryption_context: + encryption_context.update(self.encryption_context) + if additional_authenticated_data: + encryption_context.update(additional_authenticated_data) + return encryption_context + def __repr__(self): return 'KmsKeyService(key_id={0},context={1})'.format(self.key_id, self.encryption_context) diff --git a/tests/test_crypto.py b/tests/test_crypto.py index a119d03..6818d00 100644 --- a/tests/test_crypto.py +++ b/tests/test_crypto.py @@ -1,18 +1,23 @@ from __future__ import absolute_import, division, print_function, unicode_literals -import os import base64 +import json +import os + import credsmash.crypto.aes_ctr as aes_ctr import credsmash.crypto.aes_gcm as aes_gcm class DummyKeyService(object): - def generate_key_data(self, number_of_bytes): + def generate_key_data(self, number_of_bytes, additional_authenticated_data=None): key = os.urandom(int(number_of_bytes)) - return key, base64.b64encode(key) + return key, json.dumps({"key": base64.b64encode(key), "aad": additional_authenticated_data}) - def decrypt(self, encoded_key): - return base64.b64decode(encoded_key) + def decrypt(self, encoded_key, additional_authenticated_data=None): + key_data = json.loads(encoded_key) + if additional_authenticated_data != key_data['aad']: + raise RuntimeError('Mismatch additional_authenticated_data') + return base64.b64decode(key_data['key']) def test_aes_ctr_legacy(): @@ -24,20 +29,21 @@ def test_aes_ctr_legacy(): plaintext = b'abcdefghi' material = aes_ctr.seal_aes_ctr_legacy( key_service, - plaintext + plaintext, {} ) recovered_plaintext = aes_ctr.open_aes_ctr_legacy( - key_service, material + key_service, material, {} ) assert plaintext == recovered_plaintext material = aes_ctr.seal_aes_ctr_legacy( key_service, plaintext, + {}, digest_method='SHA512' ) recovered_plaintext = aes_ctr.open_aes_ctr_legacy( - key_service, material + key_service, material, {} ) assert plaintext == recovered_plaintext @@ -48,20 +54,21 @@ def test_aes_ctr(): plaintext = b'abcdefghi' material = aes_ctr.seal_aes_ctr( key_service, - plaintext + plaintext, {} ) recovered_plaintext = aes_ctr.open_aes_ctr( - key_service, material + key_service, material, {} ) assert plaintext == recovered_plaintext material = aes_ctr.seal_aes_ctr( key_service, plaintext, + {}, digest_method='SHA512' ) recovered_plaintext = aes_ctr.open_aes_ctr( - key_service, material + key_service, material, {} ) assert plaintext == recovered_plaintext @@ -72,10 +79,11 @@ def test_aes_gcm(): plaintext = b'abcdefghi' material = aes_gcm.seal_aes_gcm( key_service, - plaintext + plaintext, + {} ) recovered_plaintext = aes_gcm.open_aes_gcm( - key_service, material + key_service, material, {} ) assert plaintext == recovered_plaintext