From d61321f5e0a56dac7a51f404167048f86e4a0aff Mon Sep 17 00:00:00 2001 From: neil Date: Mon, 6 Nov 2023 15:16:07 +0000 Subject: [PATCH] adding programatic support for aws cli Profiles --- README.md | 23 +++++++++++++++++++++++ src/data_mesh_util/DataMeshAdmin.py | 13 ++++++++++--- src/data_mesh_util/DataMeshMacros.py | 13 ++++++++----- src/data_mesh_util/lib/ApiAutomator.py | 20 ++++++++++++++++---- 4 files changed, 57 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index a648e47..a453b0b 100644 --- a/README.md +++ b/README.md @@ -120,6 +120,29 @@ This file includes the following identities: For the example usage scripts, you can configure a file on your filesystem, and eference this file in through the `CredentialsFile` environment variable. For the `cli`, you can provide the path to this file using argument `--credentials-file`. Please make sure not to add this file to any publicly shared resources such as git forks of the codebase! +### Using Profiles with ~/.aws/config + +If you are using named profiles in your aws config file like the below: + +```sh +[profile someAccount-Admin] +output = json +region = eu-west-2 +credential_process = ... +``` + +You can use a Profile by name by supplying 'use_profile' instead of 'use_credentials + +```python +mesh_admin = dmu.DataMeshAdmin( + data_mesh_account_id=data_mesh_account, + region_name=aws_region, + log_level=logging.DEBUG, + use_profile='someAccount-Admin' +) +``` +This can be used to supply all account profiles, for example: mesh, produce, and consumer + ## Getting Started To install AWS Data Mesh Utils, install from Pypi: diff --git a/src/data_mesh_util/DataMeshAdmin.py b/src/data_mesh_util/DataMeshAdmin.py index 3b3162f..b67a0f3 100644 --- a/src/data_mesh_util/DataMeshAdmin.py +++ b/src/data_mesh_util/DataMeshAdmin.py @@ -35,7 +35,7 @@ class DataMeshAdmin: _automator = None def __init__(self, data_mesh_account_id: str, region_name: str = 'us-east-1', log_level: str = "INFO", - use_credentials=None): + use_credentials = None, use_profile: str = None): self._data_mesh_account_id = data_mesh_account_id # get the region for the module if region_name is None: @@ -43,10 +43,14 @@ def __init__(self, data_mesh_account_id: str, region_name: str = 'us-east-1', lo else: self._region = region_name - if use_credentials is None: + if use_credentials is not None and use_profile is not None: + raise Exception("Cannot use Credentials and Profile at the same time") + elif use_credentials is None and use_profile is None: self._session = boto3.session.Session(region_name=self._region) - else: + elif use_credentials is not None and use_profile is None: self._session = utils.create_session(credentials=use_credentials, region=self._region) + elif use_credentials is None and use_profile is not None: + self._session = boto3.session.Session(profile_name=use_profile, region_name=self._region) self._iam_client = self._session.client('iam') self._sts_client = self._session.client('sts') @@ -61,6 +65,9 @@ def __init__(self, data_mesh_account_id: str, region_name: str = 'us-east-1', lo self._automator = ApiAutomator(target_account=data_mesh_account_id, session=self._session, log_level=self._log_level) + if use_profile is not None: + self._logger.info(f"Using profile: {use_profile}") + self._logger.debug(f"Running as {self._current_identity.get('Arn')}") if self._log_level == 'DEBUG': diff --git a/src/data_mesh_util/DataMeshMacros.py b/src/data_mesh_util/DataMeshMacros.py index 1d5db99..cdc3a0d 100644 --- a/src/data_mesh_util/DataMeshMacros.py +++ b/src/data_mesh_util/DataMeshMacros.py @@ -17,13 +17,15 @@ def __init__(self, data_mesh_account_id: str, region_name: str = 'us-east-1', lo if self._log_level == 'DEBUG': utils.log_instance_signature(self, self._logger) - def bootstrap_account(self, account_type: str, mesh_credentials, account_credentials, crawler_role_arn: str = None): + def bootstrap_account(self, account_type: str, mesh_credentials=None, account_credentials=None, crawler_role_arn: str = None, + mesh_profile=None, account_profile=None): # create a data mesh admin for the mesh account mesh_admin = data_mesh_admin.DataMeshAdmin( data_mesh_account_id=self._data_mesh_account_id, region_name=self._region, log_level=self._log_level, - use_credentials=mesh_credentials + use_credentials=mesh_credentials, + use_profile=mesh_profile ) # create a data mesh admin for the target account @@ -31,14 +33,15 @@ def bootstrap_account(self, account_type: str, mesh_credentials, account_credent data_mesh_account_id=self._data_mesh_account_id, region_name=self._region, log_level=self._log_level, - use_credentials=account_credentials + use_credentials=account_credentials, + use_profile=account_profile ) if account_type.lower() == PRODUCER.lower() or account_type.lower() == self._BOTH.lower(): account_admin.initialize_producer_account() - mesh_admin.enable_account_as_producer(account_id=account_credentials.get('AccountId')) + mesh_admin.enable_account_as_producer(account_id=account_admin._sts_client.get_caller_identity()["Account"]) elif account_type.lower() == CONSUMER.lower() or account_type.lower() == self._BOTH.lower(): account_admin.initialize_consumer_account() - mesh_admin.enable_account_as_consumer(account_id=account_credentials.get('AccountId')) + mesh_admin.enable_account_as_consumer(account_id=account_admin._sts_client.get_caller_identity()["Account"]) else: raise Exception(f"Unknown Account Type {account_type}") diff --git a/src/data_mesh_util/lib/ApiAutomator.py b/src/data_mesh_util/lib/ApiAutomator.py index a7e0d94..45a8bbc 100644 --- a/src/data_mesh_util/lib/ApiAutomator.py +++ b/src/data_mesh_util/lib/ApiAutomator.py @@ -660,13 +660,25 @@ def _get_glue_resource_policy_statement_to_modify(self, region: str, policy: dic def assert_is_data_lake_admin(self, principal): lf_client = self._get_client('lakeformation') - + lf_client_admins = lf_client.get_data_lake_settings().get('DataLakeSettings').get("DataLakeAdmins") admin_matched = False - for admin in lf_client.get_data_lake_settings().get('DataLakeSettings').get("DataLakeAdmins"): - if principal == admin.get('DataLakePrincipalIdentifier'): + for admin in lf_client_admins: + admin_principal = admin.get('DataLakePrincipalIdentifier') + print(admin_principal) + if principal == admin_principal: admin_matched = True break - + if 'assumed-role' in principal: + principal_parts = principal.split(':') + admin_principal_parts = admin_principal.split(':') + if (principal_parts[0] == admin_principal_parts[0] and + principal_parts[1] == admin_principal_parts[1] and + principal_parts[3] == admin_principal_parts[3] and + principal_parts[4] == admin_principal_parts[4] and + admin_principal_parts[5] in principal_parts[5] + ): + admin_matched = True + break if admin_matched is False: raise Exception(f"Principal {principal} is not Data Lake Admin")