diff --git a/docs/config.rst b/docs/config.rst index 1fb90a337..135347bf8 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -90,6 +90,32 @@ See the `CloudFormation Limits Reference`_. .. _`CloudFormation Limits Reference`: http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/cloudformation-limits.html + +S3 Bucket tags +-------------- + +Various resources in AWS support arbitrary key-value pair tags. You can set +the `bucket_tags` Top Level Keyword to populate tags on all S3 buckets Staker +attempts to create for CloudFormation template uploads, inclduing the S3 bucket +created by the aws_lambda pre-hook. + +If bucket_tags is not set in your Configuration, stacker will fallback to the +method used to determine tags in your config by the `tags` top level keyword. +The `bucket_tags` keyword takes precedence over `tags` when applying. Example:: + + bucket_tags: + "hello": world + "my_tag:with_colons_in_key": ${dynamic_tag_value_from_my_env} + simple_tag: simple value + +If you prefer to have no tags applied to your stacks (versus the default tags +that stacker applies), specify an empty map for the top-level keyword:: + + bucket_tags: {} + +S3 Bucket Tags updates get applied on every stacker run + + Module Paths ------------ When setting the ``classpath`` for blueprints/hooks, it is sometimes desirable to @@ -137,7 +163,7 @@ The only required key for a git repository config is ``uri``, but ``branch``, commit: 12345678 If no specific commit or tag is specified for a repo, the remote repository -will be checked for newer commits on every execution of Stacker. +will be checked for newer commits on every execution of stacker. For ``.tar.gz`` & ``zip`` archives on s3, specify a ``bucket`` & ``key``:: @@ -157,7 +183,7 @@ For ``.tar.gz`` & ``zip`` archives on s3, specify a ``bucket`` & ``key``:: use_latest: false Use the ``paths`` option when subdirectories of the repo/archive should be -added to Stacker's ``sys.path``. +added to stacker's ``sys.path``. Cloned repos/archives will be cached between builds; the cache location defaults to ~/.stacker but can be manually specified via the **stacker_cache_dir** top @@ -236,23 +262,32 @@ the build action:: Tags ---- -CloudFormation supports arbitrary key-value pair tags. All stack-level, including automatically created tags, are -propagated to resources that AWS CloudFormation supports. See `AWS CloudFormation Resource Tags Type`_ for more details. -If no tags are specified, the `stacker_namespace` tag is applied to your stack with the value of `namespace` as the -tag value. +Various resources in AWS support arbitrary key-value pair tags. You can set +the `tags` Top Level Keyword to populate tags on all Resources that stacker +attempts to create via CloudFormation. All CloudFormation stack-level resources, +including automatically created tags, are propagated to resources that AWS +CloudFormation supports. See `AWS CloudFormation Resource Tags Type`_ for +more details. -If you prefer to apply a custom set of tags, specify the top-level keyword `tags` as a map. Example:: +If no tags are specified, the `stacker_namespace` tag is applied to your stack +with the value of `namespace` as the tag value. + +If you prefer to apply a custom set of tags, specify the top-level keyword +`tags` as a map. Example:: tags: "hello": world "my_tag:with_colons_in_key": ${dynamic_tag_value_from_my_env} simple_tag: simple value + If you prefer to have no tags applied to your stacks (versus the default tags that stacker applies), specify an empty map for the top-level keyword:: tags: {} +Tags updates get applied on every stacker run + .. _`AWS CloudFormation Resource Tags Type`: http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-resource-tags.html Mappings diff --git a/stacker/actions/base.py b/stacker/actions/base.py index 36c69209d..7f58cf4f6 100644 --- a/stacker/actions/base.py +++ b/stacker/actions/base.py @@ -86,7 +86,8 @@ def bucket_region(self): def ensure_cfn_bucket(self): """The CloudFormation bucket where templates will be stored.""" - ensure_s3_bucket(self.s3_conn, self.bucket_name, self.bucket_region) + ensure_s3_bucket(self.s3_conn, self.bucket_name, self.bucket_region, + self.context) def stack_template_url(self, blueprint): return stack_template_url( diff --git a/stacker/config/__init__.py b/stacker/config/__init__.py index fe1966620..471403ed9 100644 --- a/stacker/config/__init__.py +++ b/stacker/config/__init__.py @@ -351,6 +351,8 @@ class Config(Model): tags = DictType(StringType, serialize_when_none=False) + bucket_tags = DictType(StringType, serialize_when_none=False) + mappings = DictType( DictType(DictType(StringType)), serialize_when_none=False) diff --git a/stacker/context.py b/stacker/context.py index 3bcc31762..d76346652 100644 --- a/stacker/context.py +++ b/stacker/context.py @@ -102,6 +102,19 @@ def tags(self): return {"stacker_namespace": self.namespace} return {} + @property + def s3_bucket_tags(self): + s3_bucket_tags = self.config.bucket_tags + if s3_bucket_tags is not None: + return s3_bucket_tags + else: + s3_bucket_tags = self.config.tags + if s3_bucket_tags is not None: + return s3_bucket_tags + if self.namespace: + return {"stacker_namespace": self.namespace} + return {} + @property def _base_fqn(self): return self.namespace.replace(".", "-").lower() diff --git a/stacker/hooks/aws_lambda.py b/stacker/hooks/aws_lambda.py index 245d1bff2..978972901 100644 --- a/stacker/hooks/aws_lambda.py +++ b/stacker/hooks/aws_lambda.py @@ -490,7 +490,7 @@ def create_template(self): session = get_session(bucket_region) s3_client = session.client('s3') - ensure_s3_bucket(s3_client, bucket_name, bucket_region) + ensure_s3_bucket(s3_client, bucket_name, bucket_region, context) prefix = kwargs.get('prefix', '') diff --git a/stacker/tests/actions/test_base.py b/stacker/tests/actions/test_base.py index 5ac379797..78a76cc10 100644 --- a/stacker/tests/actions/test_base.py +++ b/stacker/tests/actions/test_base.py @@ -42,6 +42,17 @@ def test_ensure_cfn_bucket_exists(self): "Bucket": ANY, } ) + stubber.add_response( + "put_bucket_tagging", + service_response={}, + expected_params={ + "Bucket": ANY, + "Tagging": { + "TagSet": [ + {"Key": "stacker_namespace", + "Value": u"mynamespace"}]} + } + ) with stubber: action.ensure_cfn_bucket() @@ -65,6 +76,17 @@ def test_ensure_cfn_bucket_doesnt_exist_us_east(self): "Bucket": ANY, } ) + stubber.add_response( + "put_bucket_tagging", + service_response={}, + expected_params={ + "Bucket": ANY, + "Tagging": { + "TagSet": [ + {"Key": "stacker_namespace", + "Value": u"mynamespace"}]} + } + ) with stubber: action.ensure_cfn_bucket() @@ -91,6 +113,17 @@ def test_ensure_cfn_bucket_doesnt_exist_us_west(self): } } ) + stubber.add_response( + "put_bucket_tagging", + service_response={}, + expected_params={ + "Bucket": ANY, + "Tagging": { + "TagSet": [ + {"Key": "stacker_namespace", + "Value": u"mynamespace"}]} + } + ) with stubber: action.ensure_cfn_bucket() diff --git a/stacker/util.py b/stacker/util.py index bcc5d81d7..9e8a832a9 100644 --- a/stacker/util.py +++ b/stacker/util.py @@ -17,9 +17,12 @@ import botocore.client import botocore.exceptions + import dateutil -import yaml + from git import Repo + +import yaml from yaml.constructor import ConstructorError from yaml.nodes import MappingNode @@ -525,7 +528,7 @@ def s3_bucket_location_constraint(region): return region -def ensure_s3_bucket(s3_client, bucket_name, bucket_region): +def ensure_s3_bucket(s3_client, bucket_name, bucket_region, context): """Ensure an s3 bucket exists, if it does not then create it. Args: @@ -534,8 +537,13 @@ def ensure_s3_bucket(s3_client, bucket_name, bucket_region): bucket_name (str): The bucket being checked/created. bucket_region (str, optional): The region to create the bucket in. If not provided, will be determined by s3_client's region. + context (:class:`stacker.context.Context`): The stacker context, used + set the S3 bucket tags from the stacker config + """ + tagset = _s3_bucket_tags(context) try: + logger.debug("Checking that bucket '%s' exists.", bucket_name) s3_client.head_bucket(Bucket=bucket_name) except botocore.exceptions.ClientError as e: if e.response['Error']['Message'] == "Not Found": @@ -548,6 +556,7 @@ def ensure_s3_bucket(s3_client, bucket_name, bucket_region): create_args["CreateBucketConfiguration"] = { "LocationConstraint": location_constraint } + # pulling tags from s3_bucket_tags function s3_client.create_bucket(**create_args) elif e.response['Error']['Message'] == "Forbidden": logger.exception("Access denied for bucket %s. Did " + @@ -559,6 +568,29 @@ def ensure_s3_bucket(s3_client, bucket_name, bucket_region): bucket_name, e.response) raise + logger.debug( + "Setting tags on bucket '%s': %s", bucket_name, context.s3_bucket_tags + ) + + # setting tags on every run - must have permission to perform + # the s3:PutBucketTagging action + s3_client.put_bucket_tagging(Bucket=bucket_name, + Tagging={'TagSet': tagset}) + + +def _s3_bucket_tags(context): + """Returns the tags to be applied for a S3 bucket. + + Args: + context (:class:`stacker.context.Context`): The stacker context, used + set the S3 bucket tags from the stacker config + + Returns: + List of dictionaries containing tags to apply to that bucket. + """ + return [ + {'Key': t[0], 'Value': t[1]} for t in context.s3_bucket_tags.items()] + class Extractor(object): """Base class for extractors.""" @@ -629,8 +661,9 @@ def extension(): return '.zip' -class SourceProcessor(object): - """Makes remote python package sources available in current environment.""" +class SourceProcessor(): + """Makes remote python package sources available in the running python + environment.""" ISO8601_FORMAT = '%Y%m%dT%H%M%SZ'