From c1bb5844aabaf9df6c476510a0edc8dce94d7a65 Mon Sep 17 00:00:00 2001 From: Jeff Longland Date: Wed, 4 Sep 2024 12:06:13 -0700 Subject: [PATCH 01/42] Set target schema as a parameter --- init_table/app.py | 2 +- sync_table/app.py | 2 +- template.yaml | 16 +++++++++++++++- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/init_table/app.py b/init_table/app.py index 62a2f14..4df6f04 100644 --- a/init_table/app.py +++ b/init_table/app.py @@ -28,7 +28,7 @@ api_base_url = os.environ.get('API_BASE_URL', 'https://api-gateway.instructure.com') -namespace = 'canvas' +namespace = os.environ.get('DB_SCHEMA', 'canvas') def start(event): params = ssm_provider.get_multiple(param_path, max_age=600, decrypt=True) diff --git a/sync_table/app.py b/sync_table/app.py index 6bfae9d..394c7c0 100644 --- a/sync_table/app.py +++ b/sync_table/app.py @@ -30,7 +30,7 @@ admin_secret_arn = os.environ.get("ADMIN_SECRET_ARN") param_path = f"/{env}/canvas_data_2" api_base_url = os.environ.get("API_BASE_URL", "https://api-gateway.instructure.com") -namespace = "canvas" +namespace = os.environ.get('DB_SCHEMA', 'canvas') def start(event): params = ssm_provider.get_multiple(param_path, max_age=600, decrypt=True) diff --git a/template.yaml b/template.yaml index 4eabc94..a74d4d0 100644 --- a/template.yaml +++ b/template.yaml @@ -55,6 +55,11 @@ Parameters: Type: String Description: Username for the canvas database user. + DatabaseSchemaParameter: + Type: String + Description: Target schema for CD2 data. + Default: canvas + DatabaseMinCapacityParameter: Type: Number Description: Serverless Aurora minimum capacity. @@ -1020,6 +1025,8 @@ Resources: Value: !GetAtt AuroraDatabaseCluster.MasterUserSecret.SecretArn - Name: DB_CLUSTER_ARN Value: !GetAtt AuroraDatabaseCluster.DBClusterArn + - Name: DB_SCHEMA + Value: !Ref DatabaseSchemaParameter TimeoutSeconds: 43200 Retry: - ErrorEquals: @@ -1078,6 +1085,8 @@ Resources: Value: init_table - Name: DB_USER_SECRET_NAME Value: !Ref DatabaseUserSecretCanvas + - Name: DB_SCHEMA + Value: !Ref DatabaseSchemaParameter TimeoutSeconds: 43200 Retry: - ErrorEquals: @@ -1298,4 +1307,9 @@ Outputs: Value: !GetAtt AuroraDatabaseCluster.DBClusterArn Description: The ARN of the Canvas Data 2 Aurora cluster Export: - Name: !Sub ${ResourcePrefixParameter}-cd2-aurora-cluster-arn-${EnvironmentParameter} \ No newline at end of file + Name: !Sub ${ResourcePrefixParameter}-cd2-aurora-cluster-arn-${EnvironmentParameter} + DatabaseSchema: + Value: !Ref DatabaseSchemaParameter + Description: Target schema for CD2 data + Export: + Name: !Sub ${ResourcePrefixParameter}-cd2-schema-${EnvironmentParameter} \ No newline at end of file From 24539f0ee043d2d2ec38ff622e77f8a1658e8243 Mon Sep 17 00:00:00 2001 From: Jeff Longland Date: Wed, 4 Sep 2024 12:12:39 -0700 Subject: [PATCH 02/42] Reverted previous change. Schema is effectively the db username --- init_table/app.py | 3 +-- sync_table/app.py | 2 +- template.yaml | 16 +--------------- 3 files changed, 3 insertions(+), 18 deletions(-) diff --git a/init_table/app.py b/init_table/app.py index 4df6f04..ca07609 100644 --- a/init_table/app.py +++ b/init_table/app.py @@ -28,8 +28,6 @@ api_base_url = os.environ.get('API_BASE_URL', 'https://api-gateway.instructure.com') -namespace = os.environ.get('DB_SCHEMA', 'canvas') - def start(event): params = ssm_provider.get_multiple(param_path, max_age=600, decrypt=True) dap_client_id = params['dap_client_id'] @@ -41,6 +39,7 @@ def start(event): db_name = db_user_secret['dbname'] db_host = db_user_secret['host'] db_port = db_user_secret['port'] + namespace = db_user conn_str = f"postgresql://{db_user}:{db_password}@{db_host}:{db_port}/{db_name}?sslmode=verify-ca&sslrootcert=rds-combined-ca-bundle.pem" db_connection = DatabaseConnection(connection_string=conn_str) diff --git a/sync_table/app.py b/sync_table/app.py index 394c7c0..58dccc9 100644 --- a/sync_table/app.py +++ b/sync_table/app.py @@ -30,7 +30,6 @@ admin_secret_arn = os.environ.get("ADMIN_SECRET_ARN") param_path = f"/{env}/canvas_data_2" api_base_url = os.environ.get("API_BASE_URL", "https://api-gateway.instructure.com") -namespace = os.environ.get('DB_SCHEMA', 'canvas') def start(event): params = ssm_provider.get_multiple(param_path, max_age=600, decrypt=True) @@ -44,6 +43,7 @@ def start(event): db_name = db_user_secret["dbname"] db_host = db_user_secret["host"] db_port = db_user_secret["port"] + namespace = db_user conn_str = f"postgresql://{db_user}:{db_password}@{db_host}:{db_port}/{db_name}?sslmode=verify-ca&sslrootcert=rds-combined-ca-bundle.pem" db_connection = DatabaseConnection(connection_string=conn_str) diff --git a/template.yaml b/template.yaml index a74d4d0..4eabc94 100644 --- a/template.yaml +++ b/template.yaml @@ -55,11 +55,6 @@ Parameters: Type: String Description: Username for the canvas database user. - DatabaseSchemaParameter: - Type: String - Description: Target schema for CD2 data. - Default: canvas - DatabaseMinCapacityParameter: Type: Number Description: Serverless Aurora minimum capacity. @@ -1025,8 +1020,6 @@ Resources: Value: !GetAtt AuroraDatabaseCluster.MasterUserSecret.SecretArn - Name: DB_CLUSTER_ARN Value: !GetAtt AuroraDatabaseCluster.DBClusterArn - - Name: DB_SCHEMA - Value: !Ref DatabaseSchemaParameter TimeoutSeconds: 43200 Retry: - ErrorEquals: @@ -1085,8 +1078,6 @@ Resources: Value: init_table - Name: DB_USER_SECRET_NAME Value: !Ref DatabaseUserSecretCanvas - - Name: DB_SCHEMA - Value: !Ref DatabaseSchemaParameter TimeoutSeconds: 43200 Retry: - ErrorEquals: @@ -1307,9 +1298,4 @@ Outputs: Value: !GetAtt AuroraDatabaseCluster.DBClusterArn Description: The ARN of the Canvas Data 2 Aurora cluster Export: - Name: !Sub ${ResourcePrefixParameter}-cd2-aurora-cluster-arn-${EnvironmentParameter} - DatabaseSchema: - Value: !Ref DatabaseSchemaParameter - Description: Target schema for CD2 data - Export: - Name: !Sub ${ResourcePrefixParameter}-cd2-schema-${EnvironmentParameter} \ No newline at end of file + Name: !Sub ${ResourcePrefixParameter}-cd2-aurora-cluster-arn-${EnvironmentParameter} \ No newline at end of file From f588be3e59ea60381f14c1a54b2c5e4afd2887e7 Mon Sep 17 00:00:00 2001 From: Jeff Longland Date: Wed, 4 Sep 2024 12:18:14 -0700 Subject: [PATCH 03/42] Use parameter instead of hard-coded value --- template.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/template.yaml b/template.yaml index 4eabc94..bcaf1ca 100644 --- a/template.yaml +++ b/template.yaml @@ -259,7 +259,7 @@ Resources: DatabaseUserSecretCanvas: Type: AWS::SecretsManager::Secret Properties: - Name: !Sub ${ResourcePrefixParameter}-cd2-db-user-${EnvironmentParameter}-canvas + Name: !Sub ${ResourcePrefixParameter}-cd2-db-user-${EnvironmentParameter}-${DatabaseCd2UserParameter} Description: Database user for Canvas Data 2 canvas user GenerateSecretString: SecretStringTemplate: !Sub '{"username": "${DatabaseCd2UserParameter}"}' From 333b27f8437cb6b8f5369b47e49136fc7e482801 Mon Sep 17 00:00:00 2001 From: Jeff Longland Date: Wed, 4 Sep 2024 12:35:44 -0700 Subject: [PATCH 04/42] Make stack outputs unique in the event there are multiple copies of this stack in an account --- template.yaml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/template.yaml b/template.yaml index bcaf1ca..c66c8c8 100644 --- a/template.yaml +++ b/template.yaml @@ -1289,13 +1289,18 @@ Resources: TargetType: AWS::RDS::DBCluster Outputs: - AdminSecretArn: + DatabaseAdminSecretArn: Value: !GetAtt AuroraDatabaseCluster.MasterUserSecret.SecretArn - Description: Canvas Data 2 database admin user secret + Description: Canvas Data 2 database admin user secret ARN Export: - Name: !Sub ${ResourcePrefixParameter}-cd2-aurora-admin-secret-arn-${EnvironmentParameter} + Name: !Sub ${AWS::StackName}-${EnvironmentParameter}-db-admin-secret-arn + DatabaseUserSecretArn: + Value: !Ref DatabaseUserSecretCanvas + Description: Canvas Data 2 database user secret ARN + Export: + Name: !Sub ${AWS::StackName}-${EnvironmentParameter}-db-user-secret-arn AuroraClusterArn: Value: !GetAtt AuroraDatabaseCluster.DBClusterArn - Description: The ARN of the Canvas Data 2 Aurora cluster + Description: Canvas Data 2 Aurora cluster ARN Export: - Name: !Sub ${ResourcePrefixParameter}-cd2-aurora-cluster-arn-${EnvironmentParameter} \ No newline at end of file + Name: !Sub ${AWS::StackName}-${EnvironmentParameter}-aurora-cluster-arn \ No newline at end of file From 6769b178c88e18af11a1cd1ff6aa3812eeaa4c5e Mon Sep 17 00:00:00 2001 From: Jeff Longland Date: Wed, 4 Sep 2024 12:36:25 -0700 Subject: [PATCH 05/42] Updates to setup script to handle multiple copies of the stack in an account --- setup/prepare_aurora_db.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/setup/prepare_aurora_db.py b/setup/prepare_aurora_db.py index e9ae9c9..f29d045 100644 --- a/setup/prepare_aurora_db.py +++ b/setup/prepare_aurora_db.py @@ -9,7 +9,7 @@ parser = argparse.ArgumentParser() parser.add_argument( "--stack-name", - help="The name of the CloudFormation stack containing the Aurora database", + help="The name of the Canvas Data 2 CloudFormation stack containing the Aurora database", required=True, ) args = parser.parse_args() @@ -28,10 +28,17 @@ stack_outputs = {output["OutputKey"]: output["OutputValue"] for output in stack.outputs} stack_parameters = {parameter["ParameterKey"]: parameter["ParameterValue"] for parameter in stack.parameters} -# Get admin and cluster details -admin_secret_arn = stack_outputs["AdminSecretArn"] +# Get admin secret +admin_secret_arn = stack_outputs["DatabaseAdminSecretArn"] admin_secret = json.loads(secrets_client.get_secret_value(SecretId=admin_secret_arn)["SecretString"]) admin_username = admin_secret["username"] + +# Get CD2 database user secret +db_user_secret_arn = stack_outputs["DatabaseUserSecretArn"] +db_user_secret = json.loads(secrets_client.get_secret_value(SecretId=admin_secret_arn)["SecretString"]) +db_user_username = admin_secret["username"] + +# Get Aurora cluster ARN aurora_cluster_arn = stack_outputs["AuroraClusterArn"] # Get environment and resource prefix @@ -48,11 +55,11 @@ # Define user-role mapping user_roles = { "athena": "read_only", - "canvas": "admin" + db_user_username: "admin" } # List of usernames that should have schemas created -users_to_create_schema = ["canvas"] +users_to_create_schema = [db_user_username] def get_user_role(username): """Retrieve the role for a given username. Return read-only if not found""" @@ -144,8 +151,8 @@ def grant_user_to_admin(username, admin_username, database_name): if username in users_to_create_schema: create_schema(username, username, database_name) - # Create instructure_dap schema for canvas user with them as owner - if username == "canvas": + # Create instructure_dap schema for the CD2 database user with them as owner + if username == db_user_username: create_schema("instructure_dap", username, database_name) # Assign privileges to canvas and instructure_dap schemas From 0a3d6dc4d2e0a8591cb96dfcac501181573f9cc1 Mon Sep 17 00:00:00 2001 From: Jeff Longland Date: Wed, 4 Sep 2024 15:54:03 -0700 Subject: [PATCH 06/42] Add conditions to use an existing database --- template.yaml | 43 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/template.yaml b/template.yaml index c66c8c8..65e36c2 100644 --- a/template.yaml +++ b/template.yaml @@ -107,6 +107,16 @@ Parameters: Type: String Description: The ARN for the IAM role creating/updating this stack. + DatabaseClusterArnParameter: + Type: String + Description: (Optional) An existing Aurora database cluster ARN. Leave empty to create a new Aurora database cluster. + Default: '' + + DatabaseAdminSecretArnParameter: + Type: String + Description: (Optional) An existing database admin secret ARN. Use default value if DatabaseClusterArnParameter is empty. + Default: arn:aws:secretsmanager:region:123456789123:secret:placeholder + DatabaseSecurityGroupParameter: Type: String Description: (Optional) A security group ID for the database. Leave empty to create a new security group. @@ -114,7 +124,7 @@ Parameters: AthenaConnectorParameter: Type: String - Description: (Optional) Create an Athena connector for PostgreSQL. Default is false. + Description: (Optional) Create an Athena connector for PostgreSQL. Default is false. Must be false when values are set for DatabaseAdminSecretArnParameter and DatabaseSecurityGroupParameter. AllowedValues: [true, false] Default: false @@ -146,6 +156,14 @@ Parameters: Conditions: + # Conditions for Bring Your Own Aurora Database Cluster + ExistingDatabase: !Not [!Equals [!Ref DatabaseClusterArnParameter, '']] + ExistingDatabaseAdminSecret: !Not [!Equals [!Ref DatabaseAdminSecretArnParameter, '']] + CreateDatabase: !Not + - !Or + - Condition: ExistingDatabase + - Condition: ExistingDatabaseAdminSecret + # Conditions for Bring Your Own Data Security Group ExistingDatabaseSecurityGroup: !Not [!Equals [!Ref DatabaseSecurityGroupParameter, '']] CreateDatabaseSecurityGroup: !Equals [!Ref DatabaseSecurityGroupParameter, ''] @@ -161,7 +179,11 @@ Conditions: # Condition for optional Athena connector # Creates an Athena data source allowing CD2 data to be joined with other data sources - CreateAthenaConnector: !Equals [true, !Ref AthenaConnectorParameter] + # Can only be created if the database is created in this stack. + AthenaConnector: !Equals [true, !Ref AthenaConnectorParameter] + CreateAthenaConnector: !And + - Condition: AthenaConnector + - Condition: CreateDatabase Resources: @@ -276,7 +298,7 @@ Resources: Type: AWS::SecretsManager::SecretTargetAttachment Properties: SecretId: !Ref DatabaseUserSecretCanvas - TargetId: !Ref AuroraDatabaseCluster + TargetId: !If [ExistingDatabase, !Select [ "6", !Split [ ":" , !Ref DatabaseClusterArnParameter ] ], !Ref AuroraDatabaseCluster] TargetType: AWS::RDS::DBCluster DatabaseSubnetGroup: @@ -324,6 +346,7 @@ Resources: AuroraDatabaseCluster: Type: AWS::RDS::DBCluster + Condition: CreateDatabase Properties: Engine: aurora-postgresql EngineVersion: 15.5 @@ -364,6 +387,7 @@ Resources: AuroraDatabaseInstance: Type: AWS::RDS::DBInstance + Condition: CreateDatabase Properties: Engine: aurora-postgresql DBInstanceClass: db.serverless @@ -374,6 +398,7 @@ Resources: DatabasePostgresqlLogGroup: Type: AWS::Logs::LogGroup + Condition: CreateDatabase Properties: LogGroupName: !Sub /aws/rds/cluster/${AuroraDatabaseCluster}/postgresql RetentionInDays: !Ref LogRetentionParameter @@ -561,7 +586,7 @@ Resources: - secretsmanager:GetSecretValue Resource: - !Ref DatabaseUserSecretCanvas - - !GetAtt AuroraDatabaseCluster.MasterUserSecret.SecretArn + - !If [ExistingDatabase, !Ref DatabaseAdminSecretArnParameter, !GetAtt AuroraDatabaseCluster.MasterUserSecret.SecretArn] - PolicyName: kms PolicyDocument: Statement: @@ -589,7 +614,7 @@ Resources: Action: - rds-data:* Resource: - - !GetAtt AuroraDatabaseCluster.DBClusterArn + - !If [ExistingDatabase, !Ref DatabaseClusterArnParameter, !GetAtt AuroraDatabaseCluster.DBClusterArn] - PolicyName: state_machine PolicyDocument: Statement: @@ -1017,9 +1042,9 @@ Resources: - Name: DB_USER_SECRET_NAME Value: !Ref DatabaseUserSecretCanvas - Name: ADMIN_SECRET_ARN - Value: !GetAtt AuroraDatabaseCluster.MasterUserSecret.SecretArn + Value: !If [ExistingDatabase, !Ref DatabaseAdminSecretArnParameter, !GetAtt AuroraDatabaseCluster.MasterUserSecret.SecretArn] - Name: DB_CLUSTER_ARN - Value: !GetAtt AuroraDatabaseCluster.DBClusterArn + Value: !If [ExistingDatabase, !Ref DatabaseClusterArnParameter, !GetAtt AuroraDatabaseCluster.DBClusterArn] TimeoutSeconds: 43200 Retry: - ErrorEquals: @@ -1290,7 +1315,7 @@ Resources: Outputs: DatabaseAdminSecretArn: - Value: !GetAtt AuroraDatabaseCluster.MasterUserSecret.SecretArn + Value: !If [ExistingDatabase, !Ref DatabaseAdminSecretArnParameter, !GetAtt AuroraDatabaseCluster.MasterUserSecret.SecretArn] Description: Canvas Data 2 database admin user secret ARN Export: Name: !Sub ${AWS::StackName}-${EnvironmentParameter}-db-admin-secret-arn @@ -1300,7 +1325,7 @@ Outputs: Export: Name: !Sub ${AWS::StackName}-${EnvironmentParameter}-db-user-secret-arn AuroraClusterArn: - Value: !GetAtt AuroraDatabaseCluster.DBClusterArn + Value: !If [ExistingDatabase, !Ref DatabaseClusterArnParameter, !GetAtt AuroraDatabaseCluster.DBClusterArn] Description: Canvas Data 2 Aurora cluster ARN Export: Name: !Sub ${AWS::StackName}-${EnvironmentParameter}-aurora-cluster-arn \ No newline at end of file From 4afb8eff4bf87bc3e567cb00bce36ced76bb506e Mon Sep 17 00:00:00 2001 From: Jeff Longland Date: Wed, 4 Sep 2024 16:00:30 -0700 Subject: [PATCH 07/42] Fix copy/paste error --- setup/prepare_aurora_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/prepare_aurora_db.py b/setup/prepare_aurora_db.py index f29d045..2432f64 100644 --- a/setup/prepare_aurora_db.py +++ b/setup/prepare_aurora_db.py @@ -36,7 +36,7 @@ # Get CD2 database user secret db_user_secret_arn = stack_outputs["DatabaseUserSecretArn"] db_user_secret = json.loads(secrets_client.get_secret_value(SecretId=admin_secret_arn)["SecretString"]) -db_user_username = admin_secret["username"] +db_user_username = db_user_secret["username"] # Get Aurora cluster ARN aurora_cluster_arn = stack_outputs["AuroraClusterArn"] From ecb6909245b67a4b3c747c0abfeea859496419d4 Mon Sep 17 00:00:00 2001 From: Jeff Longland Date: Wed, 4 Sep 2024 16:01:37 -0700 Subject: [PATCH 08/42] Fix copy/paste error --- setup/prepare_aurora_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/prepare_aurora_db.py b/setup/prepare_aurora_db.py index 2432f64..35a3b0b 100644 --- a/setup/prepare_aurora_db.py +++ b/setup/prepare_aurora_db.py @@ -35,7 +35,7 @@ # Get CD2 database user secret db_user_secret_arn = stack_outputs["DatabaseUserSecretArn"] -db_user_secret = json.loads(secrets_client.get_secret_value(SecretId=admin_secret_arn)["SecretString"]) +db_user_secret = json.loads(secrets_client.get_secret_value(SecretId=db_user_secret_arn)["SecretString"]) db_user_username = db_user_secret["username"] # Get Aurora cluster ARN From b864f239aa43db0f357650cb21af1ac82c9bed95 Mon Sep 17 00:00:00 2001 From: Jeff Longland Date: Wed, 4 Sep 2024 16:10:56 -0700 Subject: [PATCH 09/42] More corrections --- template.yaml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/template.yaml b/template.yaml index 65e36c2..99ad0d3 100644 --- a/template.yaml +++ b/template.yaml @@ -112,6 +112,8 @@ Parameters: Description: (Optional) An existing Aurora database cluster ARN. Leave empty to create a new Aurora database cluster. Default: '' + # Default value is an ARN because it's referenced in the stack within a conditional statement. + # It must resolve as an ARN for the template to be valid. DatabaseAdminSecretArnParameter: Type: String Description: (Optional) An existing database admin secret ARN. Use default value if DatabaseClusterArnParameter is empty. @@ -157,8 +159,9 @@ Parameters: Conditions: # Conditions for Bring Your Own Aurora Database Cluster + # Requires both DatabaseClusterArnParameter and DatabaseAdminSecretArnParameter to be specified ExistingDatabase: !Not [!Equals [!Ref DatabaseClusterArnParameter, '']] - ExistingDatabaseAdminSecret: !Not [!Equals [!Ref DatabaseAdminSecretArnParameter, '']] + ExistingDatabaseAdminSecret: !Not [!Equals [!Ref DatabaseAdminSecretArnParameter, 'arn:aws:secretsmanager:region:123456789123:secret:placeholder']] CreateDatabase: !Not - !Or - Condition: ExistingDatabase @@ -172,14 +175,14 @@ Conditions: ExistingNotificationTopic: !Not [!Equals [!Ref NotificationTopicParameter, '']] CreateNotificationTopic: !Equals [!Ref NotificationTopicParameter, ''] - # Condition for CrowdStrike Falcon sensor + # Condition for optional CrowdStrike Falcon sensor # Configures an additional container task that injects the Falcon sensor into Fargate tasks # For more info: https://github.com/CrowdStrike/Container-Security/blob/0d2ae005be852aae403df3feb11e3bf2fd1f5258/aws-ecs/ecs-fargate-guide.md ConfigureFalconSensor: !Equals [true, !Ref FalconSensorParameter] - # Condition for optional Athena connector + # Conditions for optional Athena connector # Creates an Athena data source allowing CD2 data to be joined with other data sources - # Can only be created if the database is created in this stack. + # Requires CreateDatabase condition to be true to ensure cluster endpoint and port exist AthenaConnector: !Equals [true, !Ref AthenaConnectorParameter] CreateAthenaConnector: !And - Condition: AthenaConnector From 53c51d86b0f1a793ae2ff4c3e3f1e4ee686a1af2 Mon Sep 17 00:00:00 2001 From: Jeff Longland Date: Wed, 4 Sep 2024 16:13:25 -0700 Subject: [PATCH 10/42] Improve clarity of conditions --- template.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/template.yaml b/template.yaml index 99ad0d3..6a5c2da 100644 --- a/template.yaml +++ b/template.yaml @@ -589,7 +589,7 @@ Resources: - secretsmanager:GetSecretValue Resource: - !Ref DatabaseUserSecretCanvas - - !If [ExistingDatabase, !Ref DatabaseAdminSecretArnParameter, !GetAtt AuroraDatabaseCluster.MasterUserSecret.SecretArn] + - !If [ExistingDatabaseAdminSecret, !Ref DatabaseAdminSecretArnParameter, !GetAtt AuroraDatabaseCluster.MasterUserSecret.SecretArn] - PolicyName: kms PolicyDocument: Statement: @@ -1045,7 +1045,7 @@ Resources: - Name: DB_USER_SECRET_NAME Value: !Ref DatabaseUserSecretCanvas - Name: ADMIN_SECRET_ARN - Value: !If [ExistingDatabase, !Ref DatabaseAdminSecretArnParameter, !GetAtt AuroraDatabaseCluster.MasterUserSecret.SecretArn] + Value: !If [ExistingDatabaseAdminSecret, !Ref DatabaseAdminSecretArnParameter, !GetAtt AuroraDatabaseCluster.MasterUserSecret.SecretArn] - Name: DB_CLUSTER_ARN Value: !If [ExistingDatabase, !Ref DatabaseClusterArnParameter, !GetAtt AuroraDatabaseCluster.DBClusterArn] TimeoutSeconds: 43200 @@ -1318,7 +1318,7 @@ Resources: Outputs: DatabaseAdminSecretArn: - Value: !If [ExistingDatabase, !Ref DatabaseAdminSecretArnParameter, !GetAtt AuroraDatabaseCluster.MasterUserSecret.SecretArn] + Value: !If [ExistingDatabaseAdminSecret, !Ref DatabaseAdminSecretArnParameter, !GetAtt AuroraDatabaseCluster.MasterUserSecret.SecretArn] Description: Canvas Data 2 database admin user secret ARN Export: Name: !Sub ${AWS::StackName}-${EnvironmentParameter}-db-admin-secret-arn From ed45421c63be302c612ab353a7d970e8e552d7a2 Mon Sep 17 00:00:00 2001 From: Sung Hwang Date: Wed, 2 Jul 2025 16:10:00 -0700 Subject: [PATCH 11/42] Added AWS::StakName to the AWS resource names and stack outputs to avoid naming conflicts. --- template.yaml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/template.yaml b/template.yaml index f0bf5d7..bf93d11 100644 --- a/template.yaml +++ b/template.yaml @@ -207,7 +207,7 @@ Resources: SecretsKeyAlias: Type: AWS::KMS::Alias Properties: - AliasName: !Sub alias/${ResourcePrefixParameter}-cd2-secrets + AliasName: !Sub alias/${ResourcePrefixParameter}-${AWS::StackName}-cd2-secrets TargetKeyId: !Ref SecretsKmsKey DataKmsKey: @@ -235,7 +235,7 @@ Resources: "kms:EncryptionContext:aws:ecs:clusterAccount": - !Sub ${AWS::AccountId} "kms:EncryptionContext:aws:ecs:clusterName": - - !Sub ${ResourcePrefixParameter}-cd2-cluster + - !Sub ${ResourcePrefixParameter}-${AWS::StackName}-cd2-cluster Resource: '*' - Sid: Allow grant creation permission for Fargate tasks. Effect: Allow @@ -248,7 +248,7 @@ Resources: "kms:EncryptionContext:aws:ecs:clusterAccount": - !Sub ${AWS::AccountId} "kms:EncryptionContext:aws:ecs:clusterName": - - !Sub ${ResourcePrefixParameter}-cd2-cluster + - !Sub ${ResourcePrefixParameter}-${AWS::StackName}-cd2-cluster ForAllValues:StringEquals: "kms:GrantOperations": - "Decrypt" @@ -282,14 +282,14 @@ Resources: DataKeyAlias: Type: AWS::KMS::Alias Properties: - AliasName: !Sub alias/${ResourcePrefixParameter}-cd2-data + AliasName: !Sub alias/${ResourcePrefixParameter}-${AWS::StackName}-cd2-data TargetKeyId: !Ref DataKmsKey # Create a secret for the CD2 app "canvas" user DatabaseUserSecretCanvas: Type: AWS::SecretsManager::Secret Properties: - Name: !Sub ${ResourcePrefixParameter}-cd2-db-user-${EnvironmentParameter}-${DatabaseCd2UserParameter} + Name: !Sub ${ResourcePrefixParameter}-${AWS::StackName}-cd2-db-user-${EnvironmentParameter}-${DatabaseCd2UserParameter} Description: Database user for Canvas Data 2 canvas user GenerateSecretString: SecretStringTemplate: !Sub '{"username": "${DatabaseCd2UserParameter}"}' @@ -312,7 +312,7 @@ Resources: DatabaseSubnetGroup: Type: AWS::RDS::DBSubnetGroup Properties: - DBSubnetGroupDescription: !Sub ${ResourcePrefixParameter}-cd2-subnetgroup-${EnvironmentParameter} + DBSubnetGroupDescription: !Sub ${ResourcePrefixParameter}-${AWS::StackName}-cd2-subnetgroup-${EnvironmentParameter} SubnetIds: !Ref DatabaseSubnetListParameter Tags: - Key: !Sub ${TagNameParameter} @@ -321,7 +321,7 @@ Resources: DatabaseClientSecurityGroup: Type: AWS::EC2::SecurityGroup Properties: - GroupDescription: !Sub ${ResourcePrefixParameter}-cd2-database-client-sg-${EnvironmentParameter} + GroupDescription: !Sub ${ResourcePrefixParameter}-${AWS::StackName}-cd2-database-client-sg-${EnvironmentParameter} GroupName: Canvas Data 2 Database Client VpcId: !Ref VpcIdParameter SecurityGroupEgress: @@ -338,7 +338,7 @@ Resources: ListTablesFunctionSecurityGroup: Type: AWS::EC2::SecurityGroup Properties: - GroupDescription: !Sub ${ResourcePrefixParameter}-cd2-list-tables-function-sg-${EnvironmentParameter} + GroupDescription: !Sub ${ResourcePrefixParameter}-${AWS::StackName}-cd2-list-tables-function-sg-${EnvironmentParameter} GroupName: Canvas Data 2 ListTablesFunction VpcId: !Ref VpcIdParameter SecurityGroupEgress: @@ -356,7 +356,7 @@ Resources: Type: AWS::EC2::SecurityGroup Condition: CreateDatabaseSecurityGroup Properties: - GroupDescription: !Sub ${ResourcePrefixParameter}-cd2-database-sg-${EnvironmentParameter} + GroupDescription: !Sub ${ResourcePrefixParameter}-${AWS::StackName}-cd2-database-sg-${EnvironmentParameter} GroupName: Canvas Data 2 Database VpcId: !Ref VpcIdParameter SecurityGroupIngress: @@ -534,7 +534,7 @@ Resources: FargateCluster: Type: AWS::ECS::Cluster Properties: - ClusterName: !Sub ${ResourcePrefixParameter}-cd2-cluster + ClusterName: !Sub ${ResourcePrefixParameter}-${AWS::StackName}-cd2-cluster Configuration: ManagedStorageConfiguration: FargateEphemeralStorageKmsKeyId: !GetAtt DataKmsKey.Arn @@ -910,7 +910,7 @@ Resources: Type: AWS::SNS::Topic Condition: CreateNotificationTopic Properties: - DisplayName: !Sub "Canvas Data 2 Synchronization Workflow (${EnvironmentParameter})" + DisplayName: !Sub "Canvas Data 2 Synchronization Workflow (${AWS::StackName} - ${EnvironmentParameter})" Tags: - Key: !Sub ${TagNameParameter} Value: !Sub ${TagValueParameter} @@ -1222,14 +1222,14 @@ Resources: LambdaFunctionName: !Ref AthenaConnectorLambdaNameParameter CompositeHandler: PostGreSqlMuxCompositeHandler DefaultConnectionString: !Sub | - postgres://jdbc:postgresql://${AuroraDatabaseCluster.Endpoint.Address}:${AuroraDatabaseCluster.Endpoint.Port}/cd2?${!${ResourcePrefixParameter}-cd2-db-user-${EnvironmentParameter}-athena}&sslmode=verify-ca&sslfactory=org.postgresql.ssl.DefaultJavaSSLFactory&stringtype=unspecified + postgres://jdbc:postgresql://${AuroraDatabaseCluster.Endpoint.Address}:${AuroraDatabaseCluster.Endpoint.Port}/cd2?${!${ResourcePrefixParameter}-${AWS::StackName}-cd2-db-user-${EnvironmentParameter}-athena}&sslmode=verify-ca&sslfactory=org.postgresql.ssl.DefaultJavaSSLFactory&stringtype=unspecified DefaultScale: '0' DisableSpillEncryption: 'false' LambdaMemory: '3008' LambdaRoleARN: !GetAtt AthenaPostgreSQLConnectorRole.Arn LambdaTimeout: '900' # PermissionsBoundaryARN: - SecretNamePrefix: !Sub ${ResourcePrefixParameter}-cd2-db-user-${EnvironmentParameter}-canvas + SecretNamePrefix: !Sub ${ResourcePrefixParameter}-${AWS::StackName}-cd2-db-user-${EnvironmentParameter}-canvas SecurityGroupIds: !Ref DatabaseClientSecurityGroup SpillBucket: !Ref AthenaSpillBucket SpillPrefix: postgresql @@ -1239,7 +1239,7 @@ Resources: Type: AWS::S3::Bucket Condition: CreateAthenaConnector Properties: - BucketName: !Sub ${ResourcePrefixParameter}-athena-spill-${AWS::AccountId}-${AWS::Region} + BucketName: !Sub ${ResourcePrefixParameter}-${AWS::StackName}-athena-spill-${AWS::AccountId}-${AWS::Region} BucketEncryption: ServerSideEncryptionConfiguration: - ServerSideEncryptionByDefault: @@ -1280,7 +1280,7 @@ Resources: - Effect: Allow Action: - secretsmanager:GetSecretValue - Resource: !Sub arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${ResourcePrefixParameter}-cd2-db-user-${EnvironmentParameter}-athena* + Resource: !Sub arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${ResourcePrefixParameter}-${AWS::StackName}-cd2-db-user-${EnvironmentParameter}-athena* - Effect: Allow Action: - logs:CreateLogGroup @@ -1339,7 +1339,7 @@ Resources: Type: AWS::SecretsManager::Secret Condition: CreateAthenaConnector Properties: - Name: !Sub ${ResourcePrefixParameter}-cd2-db-user-${EnvironmentParameter}-athena + Name: !Sub ${ResourcePrefixParameter}-${AWS::StackName}-cd2-db-user-${EnvironmentParameter}-athena Description: Database user for Athena PostgreSQL connector GenerateSecretString: SecretStringTemplate: !Sub '{"username": "athena"}' From e2557cd5fb1b9f5443159a8d1ff65e500c5d741c Mon Sep 17 00:00:00 2001 From: Sung Hwang Date: Fri, 4 Jul 2025 10:23:29 -0700 Subject: [PATCH 12/42] Removed AWS::StackName from some database related AWS resources because these resources have been created already with the original names and we can reuse these resources for the other Stackset instead of creating them again for each stackset. Changed the DAP client related parameter names in the AWS Systems Manager. --- template.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/template.yaml b/template.yaml index bf93d11..ae63924 100644 --- a/template.yaml +++ b/template.yaml @@ -312,7 +312,7 @@ Resources: DatabaseSubnetGroup: Type: AWS::RDS::DBSubnetGroup Properties: - DBSubnetGroupDescription: !Sub ${ResourcePrefixParameter}-${AWS::StackName}-cd2-subnetgroup-${EnvironmentParameter} + DBSubnetGroupDescription: !Sub ${ResourcePrefixParameter}-cd2-subnetgroup-${EnvironmentParameter} SubnetIds: !Ref DatabaseSubnetListParameter Tags: - Key: !Sub ${TagNameParameter} @@ -321,7 +321,7 @@ Resources: DatabaseClientSecurityGroup: Type: AWS::EC2::SecurityGroup Properties: - GroupDescription: !Sub ${ResourcePrefixParameter}-${AWS::StackName}-cd2-database-client-sg-${EnvironmentParameter} + GroupDescription: !Sub ${ResourcePrefixParameter}-cd2-database-client-sg-${EnvironmentParameter} GroupName: Canvas Data 2 Database Client VpcId: !Ref VpcIdParameter SecurityGroupEgress: @@ -338,7 +338,7 @@ Resources: ListTablesFunctionSecurityGroup: Type: AWS::EC2::SecurityGroup Properties: - GroupDescription: !Sub ${ResourcePrefixParameter}-${AWS::StackName}-cd2-list-tables-function-sg-${EnvironmentParameter} + GroupDescription: !Sub ${ResourcePrefixParameter}-cd2-list-tables-function-sg-${EnvironmentParameter} GroupName: Canvas Data 2 ListTablesFunction VpcId: !Ref VpcIdParameter SecurityGroupEgress: @@ -356,7 +356,7 @@ Resources: Type: AWS::EC2::SecurityGroup Condition: CreateDatabaseSecurityGroup Properties: - GroupDescription: !Sub ${ResourcePrefixParameter}-${AWS::StackName}-cd2-database-sg-${EnvironmentParameter} + GroupDescription: !Sub ${ResourcePrefixParameter}-cd2-database-sg-${EnvironmentParameter} GroupName: Canvas Data 2 Database VpcId: !Ref VpcIdParameter SecurityGroupIngress: @@ -463,7 +463,7 @@ Resources: - x86_64 Policies: - SSMParameterReadPolicy: - ParameterName: !Sub '${EnvironmentParameter}/${SsmPathParameter}*' + ParameterName: !Sub '${AWS::StackName}/${SsmPathParameter}*' Environment: Variables: ENV: !Ref EnvironmentParameter @@ -514,7 +514,7 @@ Resources: - ssm:GetParameter - ssm:GetParametersByPath Resource: - - !Sub arn:aws:ssm:ca-central-1:${AWS::AccountId}:parameter/${EnvironmentParameter}/${SsmPathParameter}* + - !Sub arn:aws:ssm:ca-central-1:${AWS::AccountId}:parameter/${AWS::StackName}/${SsmPathParameter}* - PolicyName: logging PolicyDocument: Version: '2012-10-17' @@ -623,7 +623,7 @@ Resources: - ssm:GetParameter - ssm:GetParametersByPath Resource: - - !Sub arn:aws:ssm:ca-central-1:${AWS::AccountId}:parameter/${EnvironmentParameter}/${SsmPathParameter}* + - !Sub arn:aws:ssm:ca-central-1:${AWS::AccountId}:parameter/${AWS::StackName}/${SsmPathParameter}* - !If [ConfigureFalconSensor, !Sub "arn:aws:ssm:ca-central-1:${AWS::AccountId}:parameter${FalconSensorOptionsParameter}*", !Ref "AWS::NoValue"] - PolicyName: secrets PolicyDocument: From fdd880bbba2ce4f751af946ec03990ab5c3c6298 Mon Sep 17 00:00:00 2001 From: Sung Hwang Date: Mon, 7 Jul 2025 15:40:01 -0700 Subject: [PATCH 13/42] Code formatting related change --- setup/prepare_aurora_db.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/setup/prepare_aurora_db.py b/setup/prepare_aurora_db.py index 35a3b0b..e4daa09 100644 --- a/setup/prepare_aurora_db.py +++ b/setup/prepare_aurora_db.py @@ -140,17 +140,17 @@ def grant_user_to_admin(username, admin_username, database_name): secret_value = json.loads(secrets_client.get_secret_value(SecretId=secret_arn)["SecretString"]) username = secret_value["username"] database_name = secret_value["dbname"] - + # Create or update the user create_user(username, secret_value["password"], database_name) - + # Grant user to admin user grant_user_to_admin(username, admin_username, database_name) - + # Create schema for user (with them as owner) if they need a schema if username in users_to_create_schema: create_schema(username, username, database_name) - + # Create instructure_dap schema for the CD2 database user with them as owner if username == db_user_username: create_schema("instructure_dap", username, database_name) @@ -158,9 +158,9 @@ def grant_user_to_admin(username, admin_username, database_name): # Assign privileges to canvas and instructure_dap schemas # Defaults to read-only if user is not set in user_roles dict user_role = get_user_role(username) - + grant_usage_to_schema(username, "canvas", database_name) assign_privileges(username, "canvas", user_role, database_name) - + grant_usage_to_schema(username, "instructure_dap", database_name) assign_privileges(username, "instructure_dap", user_role, database_name) \ No newline at end of file From 4eb81a83f7bc72ec10362da007910dde73e8c0cb Mon Sep 17 00:00:00 2001 From: Sung Hwang Date: Tue, 8 Jul 2025 11:30:41 -0700 Subject: [PATCH 14/42] Changed the parameter name for the schema name in the create_schema_sql from username to schema_name. In the main function part, the value for the schema_name parameter for the create_schema() is already specified to use username for the schema_name. So using username parameter instead of schema_name in the create_schema_sql is redundant. Also using username as schema_name in the create_schema_sql query will cause issue when using create_schema() for creating instructure_dap schema because the function will never use instructure_dap as the schema name because it will be overrided by username. --- setup/prepare_aurora_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/prepare_aurora_db.py b/setup/prepare_aurora_db.py index e4daa09..5241655 100644 --- a/setup/prepare_aurora_db.py +++ b/setup/prepare_aurora_db.py @@ -91,7 +91,7 @@ def create_user(username, password, database_name): def create_schema(schema_name, username, database_name): """Create a schema with user as owner""" try: - create_schema_sql = f"CREATE SCHEMA IF NOT EXISTS {username} AUTHORIZATION {username}" + create_schema_sql = f"CREATE SCHEMA IF NOT EXISTS {schema_name} AUTHORIZATION {username}" execute_statement(create_schema_sql, database_name) console.print(f" - Created schema {schema_name} with owner {username}", style="bold green") except ClientError as e: From f8f82a09d20aa740319f3fdbb6d667e26be7ea99 Mon Sep 17 00:00:00 2001 From: Sung Hwang Date: Tue, 8 Jul 2025 12:09:28 -0700 Subject: [PATCH 15/42] Replaced the schema name from canvas to username because that is the schema we used when creating new schema for user. --- setup/prepare_aurora_db.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/prepare_aurora_db.py b/setup/prepare_aurora_db.py index 5241655..1c074b2 100644 --- a/setup/prepare_aurora_db.py +++ b/setup/prepare_aurora_db.py @@ -159,8 +159,8 @@ def grant_user_to_admin(username, admin_username, database_name): # Defaults to read-only if user is not set in user_roles dict user_role = get_user_role(username) - grant_usage_to_schema(username, "canvas", database_name) - assign_privileges(username, "canvas", user_role, database_name) + grant_usage_to_schema(username, username, database_name) + assign_privileges(username, username, user_role, database_name) grant_usage_to_schema(username, "instructure_dap", database_name) assign_privileges(username, "instructure_dap", user_role, database_name) \ No newline at end of file From e0dea989e99e4e990dd8a94c4a150fde09d26414 Mon Sep 17 00:00:00 2001 From: Sung Hwang Date: Tue, 8 Jul 2025 16:23:21 -0700 Subject: [PATCH 16/42] Added the necessary information when trying to deploy the CD2 stack with the existing RDS cluster. --- README.md | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3b58aba..f546703 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,25 @@ It will be helpful to have a working knowledge of AWS services and the AWS Conso By default the database will not have a public IP address and will not be accessible outside of your VPC. You will need to configure network access to the database as appropriate for your situation. +### If you want to use the existing RDS cluster + +If you want to use the existing RDS cluster, you need to create the following AWS resources: +1. One parameter in the AWS System Manager Parameter Store for Canvas Data 2 DAP Client ID +2. One parameter in the AWS System Manager Parameter Store for Canvas Data 2 DAP Client Secret + +You also need the following information: +1. RDS Database Subnets where your existing RDS cluster is located. + - Parameter Name: `DatabaseSubnetListParameter` +2. Database Admin user name for the existing RDS cluster + - Parameter Name: `DatabaseAdminUserParameter` +3. The ARN of the RDS cluster + - Parameter Name: `DatabaseClusterArnParameter` +4. The ARN of the Database Admin Secret + - Parameter Name: `DatabaseAdminUserParameter` +5. Database Security Group Name (ex: sg-.....) + - Parameter Name: `DatabaseSecurityGroupParameter` + + ## Deploying the application The Serverless Application Model Command Line Interface (SAM CLI) is an extension of the AWS CLI that adds functionality for building and testing Lambda applications. It uses Docker to run your functions in an Amazon Linux environment that matches Lambda. It can also emulate your application's build environment and API. @@ -62,7 +81,7 @@ Deploying this application will create: - An Aurora Postgres cluster - A database user credential in AWS Secrets Manager. -In order for the application to use that credential to connect to the database, a database user must be created and granted appropriate privileges. A helper script is included that will take care of this setup. +In order for the application to use that credential to connect to the database, a database user must be created and granted appropriate privileges. A helper script is included that will take care of this setup. After deploying the SAM app, run this script. You must have valid AWS credentials before running the script. From cfe16f1b2ebd9cd980f6c50d96ed70d282f069a4 Mon Sep 17 00:00:00 2001 From: Sung Hwang Date: Wed, 30 Jul 2025 14:16:01 -0700 Subject: [PATCH 17/42] Removed the stackname reference from AthenaConnector related AWS resources because the multiple CD2 related cloudformation stacks will share a single Athena connector. --- template.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/template.yaml b/template.yaml index b80e884..ce64ba9 100644 --- a/template.yaml +++ b/template.yaml @@ -1291,7 +1291,7 @@ Resources: LambdaRoleARN: !GetAtt AthenaPostgreSQLConnectorRole.Arn LambdaTimeout: '900' # PermissionsBoundaryARN: - SecretNamePrefix: !Sub ${ResourcePrefixParameter}-${AWS::StackName}-cd2-db-user-${EnvironmentParameter}-canvas + SecretNamePrefix: !Sub ${ResourcePrefixParameter}-cd2-db-user-${EnvironmentParameter}-canvas SecurityGroupIds: !Ref DatabaseClientSecurityGroup SpillBucket: !Ref AthenaSpillBucket SpillPrefix: postgresql @@ -1301,7 +1301,7 @@ Resources: Type: AWS::S3::Bucket Condition: CreateAthenaConnector Properties: - BucketName: !Sub ${ResourcePrefixParameter}-${AWS::StackName}-athena-spill-${AWS::AccountId}-${AWS::Region} + BucketName: !Sub ${ResourcePrefixParameter}-athena-spill-${AWS::AccountId}-${AWS::Region} BucketEncryption: ServerSideEncryptionConfiguration: - ServerSideEncryptionByDefault: @@ -1342,7 +1342,7 @@ Resources: - Effect: Allow Action: - secretsmanager:GetSecretValue - Resource: !Sub arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${ResourcePrefixParameter}-${AWS::StackName}-cd2-db-user-${EnvironmentParameter}-athena* + Resource: !Sub arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${ResourcePrefixParameter}-cd2-db-user-${EnvironmentParameter}-athena* - Effect: Allow Action: - logs:CreateLogGroup @@ -1401,7 +1401,7 @@ Resources: Type: AWS::SecretsManager::Secret Condition: CreateAthenaConnector Properties: - Name: !Sub ${ResourcePrefixParameter}-${AWS::StackName}-cd2-db-user-${EnvironmentParameter}-athena + Name: !Sub ${ResourcePrefixParameter}-cd2-db-user-${EnvironmentParameter}-athena Description: Database user for Athena PostgreSQL connector GenerateSecretString: SecretStringTemplate: !Sub '{"username": "athena"}' From e9351dbd7ea01cb52f6daa393a211e2c97edc36f Mon Sep 17 00:00:00 2001 From: Sung Hwang Date: Thu, 31 Jul 2025 10:24:30 -0700 Subject: [PATCH 18/42] Reverted the DAP parameter store name changes. For other canvas instance, we will use different value for the SsmPathParameter instead. --- template.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/template.yaml b/template.yaml index ce64ba9..34b11df 100644 --- a/template.yaml +++ b/template.yaml @@ -468,7 +468,7 @@ Resources: - x86_64 Policies: - SSMParameterReadPolicy: - ParameterName: !Sub '${AWS::StackName}/${SsmPathParameter}*' + ParameterName: !Sub '${EnvironmentParameter}/${SsmPathParameter}*' Environment: Variables: ENV: !Ref EnvironmentParameter @@ -640,7 +640,7 @@ Resources: - ssm:GetParameter - ssm:GetParametersByPath Resource: - - !Sub arn:aws:ssm:ca-central-1:${AWS::AccountId}:parameter/${AWS::StackName}/${SsmPathParameter}* + - !Sub arn:aws:ssm:ca-central-1:${AWS::AccountId}:parameter/${EnvironmentParameter}/${SsmPathParameter}* - !If [ConfigureFalconSensor, !Sub "arn:aws:ssm:ca-central-1:${AWS::AccountId}:parameter${FalconSensorOptionsParameter}*", !Ref "AWS::NoValue"] - PolicyName: secrets PolicyDocument: From b1a0226b615b278471ca035d1b7f05657cd5c7dc Mon Sep 17 00:00:00 2001 From: Sung Hwang Date: Thu, 31 Jul 2025 13:34:20 -0700 Subject: [PATCH 19/42] Added the condition CreateDatabaseSecurityGroup for the RDS security group related resources so that they get created under a certain condition. Added ListTablesFunctionSecurityGroupParameter and conditions associated to this parameter. ListTablesFunctionSecurityGroup only gets created under the condition. --- template.yaml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/template.yaml b/template.yaml index 34b11df..ca1b0c5 100644 --- a/template.yaml +++ b/template.yaml @@ -129,6 +129,11 @@ Parameters: Description: (Optional) A security group ID for the database. Leave empty to create a new security group. Default: '' + ListTablesFunctionSecurityGroupParameter: + Type: String + Description: (Optional) A security group ID for the ListTables Lambda function. Leave empty to create a new security group. + Default: '' + AthenaConnectorParameter: Type: String Description: (Optional) Create an Athena connector for PostgreSQL. Default is false. Must be false when values are set for DatabaseAdminSecretArnParameter and DatabaseSecurityGroupParameter. @@ -181,6 +186,10 @@ Conditions: ExistingDatabaseSecurityGroup: !Not [!Equals [!Ref DatabaseSecurityGroupParameter, '']] CreateDatabaseSecurityGroup: !Equals [!Ref DatabaseSecurityGroupParameter, ''] + # Conditions for Bring Your Own ListTables Lambda Function Security Group + ExistingListTablesFunctionSecurityGroup: !Not [!Equals [!Ref ListTablesFunctionSecurityGroupParameter, '']] + CreateListTablesFunctionSecurityGroup: !Equals [!Ref ListTablesFunctionSecurityGroupParameter, ''] + # Conditions for Bring Your Own Notification Topic ExistingNotificationTopic: !Not [!Equals [!Ref NotificationTopicParameter, '']] CreateNotificationTopic: !Equals [!Ref NotificationTopicParameter, ''] @@ -325,6 +334,7 @@ Resources: DatabaseClientSecurityGroup: Type: AWS::EC2::SecurityGroup + Condition: CreateDatabaseSecurityGroup Properties: GroupDescription: !Sub ${ResourcePrefixParameter}-cd2-database-client-sg-${EnvironmentParameter} GroupName: Canvas Data 2 Database Client @@ -342,6 +352,7 @@ Resources: ListTablesFunctionSecurityGroup: Type: AWS::EC2::SecurityGroup + Condition: CreateListTablesFunctionSecurityGroup Properties: GroupDescription: !Sub ${ResourcePrefixParameter}-cd2-list-tables-function-sg-${EnvironmentParameter} GroupName: Canvas Data 2 ListTablesFunction @@ -388,7 +399,7 @@ Resources: DatabaseClientEgressToExistingDatabase: Type: AWS::EC2::SecurityGroupEgress - Condition: ExistingDatabaseSecurityGroup + Condition: CreateDatabaseSecurityGroup Properties: GroupId: !Ref DatabaseClientSecurityGroup IpProtocol: tcp @@ -482,7 +493,7 @@ Resources: MemorySize: 256 VpcConfig: SecurityGroupIds: - - !Ref ListTablesFunctionSecurityGroup + - !If [ExistingListTablesFunctionSecurityGroup, !Ref ListTablesFunctionSecurityGroupParameter, !Ref ListTablesFunctionSecurityGroup] SubnetIds: !Ref LambdaSubnetListParameter LoggingConfig: LogGroup: !Ref ListTablesLogGroup From 5cc6aa79f272e9a58aa55cd386263e616c62ebda Mon Sep 17 00:00:00 2001 From: Sung Hwang Date: Thu, 31 Jul 2025 13:39:01 -0700 Subject: [PATCH 20/42] Added DatabaseClientSecurityGroupParameter and the condition associated with it. This allows any resource referencing this security group can reference the existing security group if existed. --- template.yaml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/template.yaml b/template.yaml index ca1b0c5..b6f7648 100644 --- a/template.yaml +++ b/template.yaml @@ -129,6 +129,11 @@ Parameters: Description: (Optional) A security group ID for the database. Leave empty to create a new security group. Default: '' + DatabaseClientSecurityGroupParameter: + Type: String + Description: (Optional) A security group ID for the DatabaseClient. Leave empty to create a new security group. + Default: '' + ListTablesFunctionSecurityGroupParameter: Type: String Description: (Optional) A security group ID for the ListTables Lambda function. Leave empty to create a new security group. @@ -186,6 +191,9 @@ Conditions: ExistingDatabaseSecurityGroup: !Not [!Equals [!Ref DatabaseSecurityGroupParameter, '']] CreateDatabaseSecurityGroup: !Equals [!Ref DatabaseSecurityGroupParameter, ''] + # Conditions for Bring Your Own Database Client Security Group + ExistingDatabaseClientSecurityGroup: !Not [!Equals [!Ref DatabaseClientSecurityGroupParameter, '']] + # Conditions for Bring Your Own ListTables Lambda Function Security Group ExistingListTablesFunctionSecurityGroup: !Not [!Equals [!Ref ListTablesFunctionSecurityGroupParameter, '']] CreateListTablesFunctionSecurityGroup: !Equals [!Ref ListTablesFunctionSecurityGroupParameter, ''] @@ -1147,7 +1155,7 @@ Resources: AwsvpcConfiguration: AssignPublicIp: DISABLED SecurityGroups: - - !Ref DatabaseClientSecurityGroup + - !If [ExistingDatabaseClientSecurityGroup, !Ref DatabaseClientSecurityGroupParameter, !Ref DatabaseClientSecurityGroup] Subnets: - Fn::ImportValue: !Sub ${ResourcePrefixParameter}-vpc--privateSubnetA - Fn::ImportValue: !Sub ${ResourcePrefixParameter}-vpc--privateSubnetB @@ -1209,7 +1217,7 @@ Resources: AwsvpcConfiguration: AssignPublicIp: DISABLED SecurityGroups: - - !Ref DatabaseClientSecurityGroup + - !If [ExistingDatabaseClientSecurityGroup, !Ref DatabaseClientSecurityGroupParameter, !Ref DatabaseClientSecurityGroup] Subnets: - Fn::ImportValue: !Sub ${ResourcePrefixParameter}-vpc--privateSubnetA - Fn::ImportValue: !Sub ${ResourcePrefixParameter}-vpc--privateSubnetB From c6323ac84f5a38fb5b84ec959748d9b37caacb82 Mon Sep 17 00:00:00 2001 From: Sung Hwang Date: Fri, 1 Aug 2025 11:08:01 -0700 Subject: [PATCH 21/42] Added an option to get security group ID of DatabaseClientSecurityGroup from DatabaseClientSecurityGroupParameter for AthenaPostgreSQLConnector if it already exists. --- template.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/template.yaml b/template.yaml index b6f7648..46d072a 100644 --- a/template.yaml +++ b/template.yaml @@ -1311,7 +1311,8 @@ Resources: LambdaTimeout: '900' # PermissionsBoundaryARN: SecretNamePrefix: !Sub ${ResourcePrefixParameter}-cd2-db-user-${EnvironmentParameter}-canvas - SecurityGroupIds: !Ref DatabaseClientSecurityGroup + SecurityGroupIds: + - !If [ExistingDatabaseClientSecurityGroup, !Ref DatabaseClientSecurityGroupParameter, !Ref DatabaseClientSecurityGroup] SpillBucket: !Ref AthenaSpillBucket SpillPrefix: postgresql SubnetIds: !Join [ ",", !Ref LambdaSubnetListParameter ] From 0df8ba05f2a2315ebd658a02c9edc5f651d0f620 Mon Sep 17 00:00:00 2001 From: Sung Hwang Date: Fri, 1 Aug 2025 13:33:39 -0700 Subject: [PATCH 22/42] Wrapped all Ref calls to DatabaseClientSecurityGroup with an Fn::If that falls back to DatabaseClientSecurityGroupParameter. --- template.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/template.yaml b/template.yaml index 46d072a..ac52732 100644 --- a/template.yaml +++ b/template.yaml @@ -387,7 +387,7 @@ Resources: - IpProtocol: tcp FromPort: 5432 ToPort: 5432 - SourceSecurityGroupId: !Ref DatabaseClientSecurityGroup + SourceSecurityGroupId: !If [ExistingDatabaseClientSecurityGroup, !Ref DatabaseClientSecurityGroupParameter, !Ref DatabaseClientSecurityGroup] # TODO: add more ingress rules based on parameter(s) Tags: - Key: Name @@ -399,7 +399,7 @@ Resources: Type: AWS::EC2::SecurityGroupEgress Condition: CreateDatabaseSecurityGroup Properties: - GroupId: !Ref DatabaseClientSecurityGroup + GroupId: !If [ExistingDatabaseClientSecurityGroup, !Ref DatabaseClientSecurityGroupParameter, !Ref DatabaseClientSecurityGroup] IpProtocol: tcp FromPort: 5432 ToPort: 5432 @@ -409,7 +409,7 @@ Resources: Type: AWS::EC2::SecurityGroupEgress Condition: CreateDatabaseSecurityGroup Properties: - GroupId: !Ref DatabaseClientSecurityGroup + GroupId: !If [ExistingDatabaseClientSecurityGroup, !Ref DatabaseClientSecurityGroupParameter, !Ref DatabaseClientSecurityGroup] IpProtocol: tcp FromPort: 5432 ToPort: 5432 From 9a8befe80ad5a08b2ca3d2feb4fb496cb55a69a5 Mon Sep 17 00:00:00 2001 From: Sung Hwang Date: Fri, 1 Aug 2025 15:12:20 -0700 Subject: [PATCH 23/42] Added a new condition called CreateDatabaseClientSecurityGroup and applied to the DatabaseClientSecurityGroup AWS resource. --- template.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/template.yaml b/template.yaml index ac52732..8e38731 100644 --- a/template.yaml +++ b/template.yaml @@ -193,6 +193,7 @@ Conditions: # Conditions for Bring Your Own Database Client Security Group ExistingDatabaseClientSecurityGroup: !Not [!Equals [!Ref DatabaseClientSecurityGroupParameter, '']] + CreateDatabaseClientSecurityGroup: !Equals [!Ref DatabaseClientSecurityGroupParameter, ''] # Conditions for Bring Your Own ListTables Lambda Function Security Group ExistingListTablesFunctionSecurityGroup: !Not [!Equals [!Ref ListTablesFunctionSecurityGroupParameter, '']] @@ -342,7 +343,7 @@ Resources: DatabaseClientSecurityGroup: Type: AWS::EC2::SecurityGroup - Condition: CreateDatabaseSecurityGroup + Condition: CreateDatabaseClientSecurityGroup Properties: GroupDescription: !Sub ${ResourcePrefixParameter}-cd2-database-client-sg-${EnvironmentParameter} GroupName: Canvas Data 2 Database Client From 8b4c669317545b3d15c23bd6e27545c3db10324e Mon Sep 17 00:00:00 2001 From: Sung Hwang Date: Tue, 5 Aug 2025 10:39:10 -0700 Subject: [PATCH 24/42] Changed the SecurityGroupIds expression for AthenaPostgreSQLConnector to fix the cloudformation error. --- template.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/template.yaml b/template.yaml index 8e38731..6deff66 100644 --- a/template.yaml +++ b/template.yaml @@ -1312,8 +1312,10 @@ Resources: LambdaTimeout: '900' # PermissionsBoundaryARN: SecretNamePrefix: !Sub ${ResourcePrefixParameter}-cd2-db-user-${EnvironmentParameter}-canvas - SecurityGroupIds: - - !If [ExistingDatabaseClientSecurityGroup, !Ref DatabaseClientSecurityGroupParameter, !Ref DatabaseClientSecurityGroup] + SecurityGroupIds: !If + - ExistingDatabaseClientSecurityGroup + - !Ref DatabaseClientSecurityGroupParameter + - !Ref DatabaseClientSecurityGroup SpillBucket: !Ref AthenaSpillBucket SpillPrefix: postgresql SubnetIds: !Join [ ",", !Ref LambdaSubnetListParameter ] From be79d5ecd13a794af63a8b741ad0e3719a34dc93 Mon Sep 17 00:00:00 2001 From: Sung Hwang Date: Tue, 5 Aug 2025 13:49:50 -0700 Subject: [PATCH 25/42] Added the conditions for the resource creation for DatabaseSubnetGroup and DatabaseClientEgressToExistingDatabase. --- template.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/template.yaml b/template.yaml index 6deff66..aa9dcd7 100644 --- a/template.yaml +++ b/template.yaml @@ -334,6 +334,7 @@ Resources: DatabaseSubnetGroup: Type: AWS::RDS::DBSubnetGroup + Condition: CreateDatabase Properties: DBSubnetGroupDescription: !Sub ${ResourcePrefixParameter}-cd2-subnetgroup-${EnvironmentParameter} SubnetIds: !Ref DatabaseSubnetListParameter @@ -408,7 +409,7 @@ Resources: DatabaseClientEgressToExistingDatabase: Type: AWS::EC2::SecurityGroupEgress - Condition: CreateDatabaseSecurityGroup + Condition: CreateDatabaseClientSecurityGroup Properties: GroupId: !If [ExistingDatabaseClientSecurityGroup, !Ref DatabaseClientSecurityGroupParameter, !Ref DatabaseClientSecurityGroup] IpProtocol: tcp From f66be14b6e021e1d908cb0d52a159ffea0b1be48 Mon Sep 17 00:00:00 2001 From: Sung Hwang Date: Tue, 5 Aug 2025 14:31:29 -0700 Subject: [PATCH 26/42] Removed the resource DatabaseClientEgressToExistingDatabase because it is not needed. Changed the condition for DatabaseClientEgressToDatabase so that it only gets creaed when your stack set needs to create DatabaseClientSecurityGroup. --- template.yaml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/template.yaml b/template.yaml index aa9dcd7..98221f9 100644 --- a/template.yaml +++ b/template.yaml @@ -399,7 +399,6 @@ Resources: DatabaseClientEgressToDatabase: Type: AWS::EC2::SecurityGroupEgress - Condition: CreateDatabaseSecurityGroup Properties: GroupId: !If [ExistingDatabaseClientSecurityGroup, !Ref DatabaseClientSecurityGroupParameter, !Ref DatabaseClientSecurityGroup] IpProtocol: tcp @@ -407,16 +406,6 @@ Resources: ToPort: 5432 DestinationSecurityGroupId: !GetAtt DatabaseSecurityGroup.GroupId - DatabaseClientEgressToExistingDatabase: - Type: AWS::EC2::SecurityGroupEgress - Condition: CreateDatabaseClientSecurityGroup - Properties: - GroupId: !If [ExistingDatabaseClientSecurityGroup, !Ref DatabaseClientSecurityGroupParameter, !Ref DatabaseClientSecurityGroup] - IpProtocol: tcp - FromPort: 5432 - ToPort: 5432 - DestinationSecurityGroupId: !Ref DatabaseSecurityGroupParameter - AuroraDatabaseCluster: Type: AWS::RDS::DBCluster Condition: CreateDatabase From 87cec2519d3a34e0b5f7ed01516e73e66afb1d83 Mon Sep 17 00:00:00 2001 From: Sung Hwang Date: Tue, 5 Aug 2025 14:41:38 -0700 Subject: [PATCH 27/42] Revert "Removed the resource DatabaseClientEgressToExistingDatabase because it is not needed. Changed the condition for DatabaseClientEgressToDatabase so that it only gets creaed when your stack set needs to create DatabaseClientSecurityGroup." This reverts commit f66be14b6e021e1d908cb0d52a159ffea0b1be48. --- template.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/template.yaml b/template.yaml index 98221f9..aa9dcd7 100644 --- a/template.yaml +++ b/template.yaml @@ -399,6 +399,7 @@ Resources: DatabaseClientEgressToDatabase: Type: AWS::EC2::SecurityGroupEgress + Condition: CreateDatabaseSecurityGroup Properties: GroupId: !If [ExistingDatabaseClientSecurityGroup, !Ref DatabaseClientSecurityGroupParameter, !Ref DatabaseClientSecurityGroup] IpProtocol: tcp @@ -406,6 +407,16 @@ Resources: ToPort: 5432 DestinationSecurityGroupId: !GetAtt DatabaseSecurityGroup.GroupId + DatabaseClientEgressToExistingDatabase: + Type: AWS::EC2::SecurityGroupEgress + Condition: CreateDatabaseClientSecurityGroup + Properties: + GroupId: !If [ExistingDatabaseClientSecurityGroup, !Ref DatabaseClientSecurityGroupParameter, !Ref DatabaseClientSecurityGroup] + IpProtocol: tcp + FromPort: 5432 + ToPort: 5432 + DestinationSecurityGroupId: !Ref DatabaseSecurityGroupParameter + AuroraDatabaseCluster: Type: AWS::RDS::DBCluster Condition: CreateDatabase From 2f9b6440421899835b61ab682213f47b861ae0d8 Mon Sep 17 00:00:00 2001 From: Sung Hwang Date: Wed, 6 Aug 2025 11:05:10 -0700 Subject: [PATCH 28/42] Created new parameters for SecretsKmsKey and DataKmsKey. Added if-conditions for several AWS resources to reference KMS Key ID or ARN value either from the actual KMS key resource or parameters. --- template.yaml | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/template.yaml b/template.yaml index aa9dcd7..be3a149 100644 --- a/template.yaml +++ b/template.yaml @@ -176,6 +176,26 @@ Parameters: Description: The secret name for Slack Incoming Webhook URL for the notification Default: '' + SecretsKmsKeyIDParameter: + Type: String + Description: (Optional) The key ID for existing SecretsKmsKey. Leave empty to create a new KMS key. + Default: '' + + SecretsKmsKeyArnParameter: + Type: String + Description: (Optional) The ARN for existing SecretsKmsKey. Leave empty to create a new KMS key. + Default: arn:aws:kms:region:123456789123:key:placeholder + + DataKmsKeyIDParameter: + Type: String + Description: (Optional) The Key ID for DataKmsKey. Leave empty to create a new KMS key. + Default: '' + + DataKmsKeyArnParameter: + Type: String + Description: (Optional) The ARN for existing DataKmsKey. Leave empty to create a new KMS key. + Default: arn:aws:kms:region:123456789123:key:placeholder + Conditions: # Conditions for Bring Your Own Aurora Database Cluster @@ -220,6 +240,7 @@ Resources: SecretsKmsKey: Type: AWS::KMS::Key + Condition: CreateDatabase Properties: Description: Key for encrypting Canvas Data 2 secrets Enabled: true @@ -229,12 +250,14 @@ Resources: SecretsKeyAlias: Type: AWS::KMS::Alias + Condition: CreateDatabase Properties: AliasName: !Sub alias/${ResourcePrefixParameter}-${AWS::StackName}-cd2-secrets TargetKeyId: !Ref SecretsKmsKey DataKmsKey: Type: AWS::KMS::Key + Condition: CreateDatabase Properties: Description: Key for encrypting Canvas Data 2 data Enabled: true @@ -304,6 +327,7 @@ Resources: DataKeyAlias: Type: AWS::KMS::Alias + Condition: CreateDatabase Properties: AliasName: !Sub alias/${ResourcePrefixParameter}-${AWS::StackName}-cd2-data TargetKeyId: !Ref DataKmsKey @@ -319,7 +343,7 @@ Resources: GenerateStringKey: password PasswordLength: 24 ExcludePunctuation: true - KmsKeyId: !Ref SecretsKmsKey + KmsKeyId: !If [CreateDatabase, !Ref SecretsKmsKey, !Ref SecretsKmsKeyIDParameter] Tags: - Key: !Sub ${TagNameParameter} Value: !Sub ${TagValueParameter} @@ -575,7 +599,7 @@ Resources: ClusterName: !Sub ${ResourcePrefixParameter}-${AWS::StackName}-cd2-cluster Configuration: ManagedStorageConfiguration: - FargateEphemeralStorageKmsKeyId: !GetAtt DataKmsKey.Arn + FargateEphemeralStorageKmsKeyId: !If [CreateDatabase, !GetAtt DataKmsKey.Arn, !Ref DataKmsKeyArnParameter] Tags: - Key: !Sub ${TagNameParameter} Value: !Sub ${TagValueParameter} @@ -614,7 +638,7 @@ Resources: Action: - kms:* Resource: - - !GetAtt DataKmsKey.Arn + - !If [CreateDatabase, !GetAtt DataKmsKey.Arn, !Ref DataKmsKeyArnParameter] - !If - ConfigureFalconSensor - PolicyName: falcon_parameter_store @@ -678,7 +702,7 @@ Resources: - Effect: Allow Action: - kms:Decrypt - Resource: !GetAtt SecretsKmsKey.Arn + Resource: !If [CreateDatabase, !GetAtt SecretsKmsKey.Arn, !Ref SecretsKmsKeyArnParameter] - PolicyName: ecr PolicyDocument: Statement: @@ -719,7 +743,7 @@ Resources: Action: - kms:* Resource: - - !GetAtt DataKmsKey.Arn + - !If [CreateDatabase, !GetAtt DataKmsKey.Arn, !Ref DataKmsKeyArnParameter] - PolicyName: logging PolicyDocument: Version: '2012-10-17' From 1b35d9c323ce3c0e3f22924b07fba5386ff01f28 Mon Sep 17 00:00:00 2001 From: Sung Hwang Date: Thu, 7 Aug 2025 10:02:41 -0700 Subject: [PATCH 29/42] Removed SecretsKmsKeyArnParameter and DataKmsKeyArnParameter because there are too many parameters for canvas-catlog in the codepipeline step that causes the error during the canvas-data-2-codepipeline cloudformation stack set udpate. Replaced these parameters by composing the KMS Arn from various parameters. --- template.yaml | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/template.yaml b/template.yaml index be3a149..1256ffa 100644 --- a/template.yaml +++ b/template.yaml @@ -181,21 +181,11 @@ Parameters: Description: (Optional) The key ID for existing SecretsKmsKey. Leave empty to create a new KMS key. Default: '' - SecretsKmsKeyArnParameter: - Type: String - Description: (Optional) The ARN for existing SecretsKmsKey. Leave empty to create a new KMS key. - Default: arn:aws:kms:region:123456789123:key:placeholder - DataKmsKeyIDParameter: Type: String Description: (Optional) The Key ID for DataKmsKey. Leave empty to create a new KMS key. Default: '' - DataKmsKeyArnParameter: - Type: String - Description: (Optional) The ARN for existing DataKmsKey. Leave empty to create a new KMS key. - Default: arn:aws:kms:region:123456789123:key:placeholder - Conditions: # Conditions for Bring Your Own Aurora Database Cluster @@ -599,7 +589,7 @@ Resources: ClusterName: !Sub ${ResourcePrefixParameter}-${AWS::StackName}-cd2-cluster Configuration: ManagedStorageConfiguration: - FargateEphemeralStorageKmsKeyId: !If [CreateDatabase, !GetAtt DataKmsKey.Arn, !Ref DataKmsKeyArnParameter] + FargateEphemeralStorageKmsKeyId: !If [CreateDatabase, !GetAtt DataKmsKey.Arn, !Sub "arn:aws:kms:${AWS::Region}:${AWS::AccountId}:key/${DataKmsKeyIDParameter}"] Tags: - Key: !Sub ${TagNameParameter} Value: !Sub ${TagValueParameter} @@ -638,7 +628,7 @@ Resources: Action: - kms:* Resource: - - !If [CreateDatabase, !GetAtt DataKmsKey.Arn, !Ref DataKmsKeyArnParameter] + - !If [CreateDatabase, !GetAtt DataKmsKey.Arn, !Sub "arn:aws:kms:${AWS::Region}:${AWS::AccountId}:key/${DataKmsKeyIDParameter}"] - !If - ConfigureFalconSensor - PolicyName: falcon_parameter_store @@ -702,7 +692,7 @@ Resources: - Effect: Allow Action: - kms:Decrypt - Resource: !If [CreateDatabase, !GetAtt SecretsKmsKey.Arn, !Ref SecretsKmsKeyArnParameter] + Resource: !If [CreateDatabase, !GetAtt SecretsKmsKey.Arn, !Sub "arn:aws:kms:${AWS::Region}:${AWS::AccountId}:key/${SecretsKmsKeyIDParameter}"] - PolicyName: ecr PolicyDocument: Statement: @@ -743,7 +733,7 @@ Resources: Action: - kms:* Resource: - - !If [CreateDatabase, !GetAtt DataKmsKey.Arn, !Ref DataKmsKeyArnParameter] + - !If [CreateDatabase, !GetAtt DataKmsKey.Arn, !Sub "arn:aws:kms:${AWS::Region}:${AWS::AccountId}:key/${DataKmsKeyIDParameter}"] - PolicyName: logging PolicyDocument: Version: '2012-10-17' From a09c26bafcd490b0bfe1e57d00a2231bad26c3c2 Mon Sep 17 00:00:00 2001 From: Sung Hwang Date: Thu, 7 Aug 2025 12:41:20 -0700 Subject: [PATCH 30/42] Added the hardcoded ECS fargate cluster name for the canvas-data-2-catalog under the DataKmsKey. --- template.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/template.yaml b/template.yaml index 1256ffa..baba266 100644 --- a/template.yaml +++ b/template.yaml @@ -270,8 +270,10 @@ Resources: StringEquals: "kms:EncryptionContext:aws:ecs:clusterAccount": - !Sub ${AWS::AccountId} + ForAnyValue:StringEquals: "kms:EncryptionContext:aws:ecs:clusterName": - !Sub ${ResourcePrefixParameter}-${AWS::StackName}-cd2-cluster + - !Sub ${ResourcePrefixParameter}-ubcla-canvas-data-2-catalog-cd2-cluster Resource: '*' - Sid: Allow grant creation permission for Fargate tasks. Effect: Allow @@ -283,8 +285,10 @@ Resources: StringEquals: "kms:EncryptionContext:aws:ecs:clusterAccount": - !Sub ${AWS::AccountId} + ForAnyValue:StringEquals: "kms:EncryptionContext:aws:ecs:clusterName": - !Sub ${ResourcePrefixParameter}-${AWS::StackName}-cd2-cluster + - !Sub ${ResourcePrefixParameter}-ubcla-canvas-data-2-catalog-cd2-cluster ForAllValues:StringEquals: "kms:GrantOperations": - "Decrypt" From 03e74a447e428f96de483bd6a879868dab23300e Mon Sep 17 00:00:00 2001 From: Sung Hwang Date: Thu, 7 Aug 2025 16:29:22 -0700 Subject: [PATCH 31/42] Assigned the dynamic name for the CD2 Data Refresh event schedule. --- template.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/template.yaml b/template.yaml index baba266..4615763 100644 --- a/template.yaml +++ b/template.yaml @@ -1129,6 +1129,7 @@ Resources: Type: ScheduleV2 Properties: Description: Execute the Canvas Data 2 Step Function every 3 hours + Name: !Sub ${AWS::StackName}-CD2-Refresh-schedule FlexibleTimeWindow: Mode: FLEXIBLE MaximumWindowInMinutes: 10 From 6bee27ce14da1cb76d462bf29e135f7c4d37691f Mon Sep 17 00:00:00 2001 From: Sung Hwang Date: Mon, 11 Aug 2025 17:04:01 -0700 Subject: [PATCH 32/42] Added the System Managers parameter name in the inti_table, list_tables, and sync_table code so that it can reference the correct SSM parameters in case you have multiple CD2 cloudfomration stack. --- init_table/app.py | 5 +++-- list_tables/app.py | 3 ++- sync_table/app.py | 3 ++- template.yaml | 5 +++++ 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/init_table/app.py b/init_table/app.py index ca07609..697fd02 100644 --- a/init_table/app.py +++ b/init_table/app.py @@ -22,9 +22,10 @@ logger = Logger() env = os.environ.get('ENV', 'dev') +ssm_parameter_name = os.environ.get('SSM_PARAMETER_NAME', 'canvas_data_2') db_user_secret_name = os.environ.get('DB_USER_SECRET_NAME') -param_path = f'/{env}/canvas_data_2' +param_path = f'/{env}/{ssm_parameter_name}' api_base_url = os.environ.get('API_BASE_URL', 'https://api-gateway.instructure.com') @@ -95,7 +96,7 @@ async def init_table(credentials, api_base_url, db_connection, namespace, table_ if token: stepfunctions.send_task_success( taskToken=token, - output=json.dumps(payload)) + output=json.dumps(payload)) """ if token and result['state'] == 'complete': diff --git a/list_tables/app.py b/list_tables/app.py index 3f91d5d..7870c8d 100644 --- a/list_tables/app.py +++ b/list_tables/app.py @@ -17,8 +17,9 @@ logger = Logger() env = os.environ.get('ENV', 'dev') +ssm_parameter_name = os.environ.get('SSM_PARAMETER_NAME', 'canvas_data_2') -param_path = f'/{env}/canvas_data_2' +param_path = f'/{env}/{ssm_parameter_name}' api_base_url = os.environ.get('API_BASE_URL', 'https://api-gateway.instructure.com') diff --git a/sync_table/app.py b/sync_table/app.py index ad84d8d..64fb2e7 100644 --- a/sync_table/app.py +++ b/sync_table/app.py @@ -29,7 +29,8 @@ db_cluster_arn = os.environ.get("DB_CLUSTER_ARN") db_user_secret_name = os.environ.get("DB_USER_SECRET_NAME") admin_secret_arn = os.environ.get("ADMIN_SECRET_ARN") -param_path = f"/{env}/canvas_data_2" +ssm_parameter_name = os.environ.get('SSM_PARAMETER_NAME', 'canvas_data_2') +param_path = f"/{env}/{ssm_parameter_name}" api_base_url = os.environ.get("API_BASE_URL", "https://api-gateway.instructure.com") FUNCTION_NAME = 'sync_table' diff --git a/template.yaml b/template.yaml index 4615763..4ec7065 100644 --- a/template.yaml +++ b/template.yaml @@ -517,6 +517,7 @@ Resources: SKIP_TABLES: !Ref SkipTablesParameter SLACK_WEBHOOK_SECRET_NAME: !Sub ${SlackWebHookURLSecretNameParameter} STACK_NAME: !Sub ${AWS::StackName} + SSM_PARAMETER_NAME: !Sub ${SsmPathParameter} Timeout: 120 MemorySize: 256 VpcConfig: @@ -1202,6 +1203,8 @@ Resources: Value: !If [ExistingDatabaseAdminSecret, !Ref DatabaseAdminSecretArnParameter, !GetAtt AuroraDatabaseCluster.MasterUserSecret.SecretArn] - Name: DB_CLUSTER_ARN Value: !If [ExistingDatabase, !Ref DatabaseClusterArnParameter, !GetAtt AuroraDatabaseCluster.DBClusterArn] + - Name: SSM_PARAMETER_NAME + Value: !Sub ${SsmPathParameter} TimeoutSeconds: 43200 Retry: - ErrorEquals: @@ -1260,6 +1263,8 @@ Resources: Value: init_table - Name: DB_USER_SECRET_NAME Value: !Ref DatabaseUserSecretCanvas + - Name: SSM_PARAMETER_NAME + Value: !Sub ${SsmPathParameter} TimeoutSeconds: 43200 Retry: - ErrorEquals: From e15150170697ad675925c30d578734591aea1e18 Mon Sep 17 00:00:00 2001 From: Sung Hwang Date: Mon, 11 Aug 2025 17:45:01 -0700 Subject: [PATCH 33/42] Added a new environment variable called DB_CD2_USER to the list_tables function so that it can assign a dynamic value for the namespace. --- template.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/template.yaml b/template.yaml index 4ec7065..560d463 100644 --- a/template.yaml +++ b/template.yaml @@ -518,6 +518,7 @@ Resources: SLACK_WEBHOOK_SECRET_NAME: !Sub ${SlackWebHookURLSecretNameParameter} STACK_NAME: !Sub ${AWS::StackName} SSM_PARAMETER_NAME: !Sub ${SsmPathParameter} + DB_CD2_USER: !Sub ${DatabaseCd2UserParameter} Timeout: 120 MemorySize: 256 VpcConfig: From 6f9eb3985f58efced561822c91825f7d8d942a48 Mon Sep 17 00:00:00 2001 From: Sung Hwang Date: Mon, 11 Aug 2025 17:45:50 -0700 Subject: [PATCH 34/42] Applied the new environment variable to the list_tables code. --- list_tables/app.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/list_tables/app.py b/list_tables/app.py index 7870c8d..42dbd59 100644 --- a/list_tables/app.py +++ b/list_tables/app.py @@ -18,12 +18,13 @@ env = os.environ.get('ENV', 'dev') ssm_parameter_name = os.environ.get('SSM_PARAMETER_NAME', 'canvas_data_2') +db_user = os.environ.get('DB_CD2_USER', 'canvas') param_path = f'/{env}/{ssm_parameter_name}' api_base_url = os.environ.get('API_BASE_URL', 'https://api-gateway.instructure.com') -namespace = 'canvas' +namespace = db_user REGION = os.environ["AWS_REGION"] SLACK_WEBHOOK_URL_SECRET_NAME = os.getenv("SLACK_WEBHOOK_SECRET_NAME") From 026932a72460a2c41240752a9bb88f79bcfb3f71 Mon Sep 17 00:00:00 2001 From: Sung Hwang Date: Tue, 12 Aug 2025 15:22:09 -0700 Subject: [PATCH 35/42] Added MainCD2StackNameParameter. This parameter is used in the ECS Task Definition to reference the ECR images from the main CD2 stack for the other canvas CD2 stack set. --- template.yaml | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/template.yaml b/template.yaml index 560d463..e8cba56 100644 --- a/template.yaml +++ b/template.yaml @@ -186,6 +186,11 @@ Parameters: Description: (Optional) The Key ID for DataKmsKey. Leave empty to create a new KMS key. Default: '' + MainCD2StackNameParameter: + Type: String + Description: (Optional) The Cloudformation stack name of the main Canvas Data 2 Stack. + Default: 'ubcla-canvas-data-2' + Conditions: # Conditions for Bring Your Own Aurora Database Cluster @@ -776,7 +781,11 @@ Resources: - Name: !Sub ${AWS::StackName}-InitTable Cpu: !Ref TaskCpuParameter Memory: !Ref TaskMemoryParameter - Image: !Sub ${EcrAccountNumberParameter}.dkr.ecr.${AWS::Region}.amazonaws.com/${AWS::StackName}/init-table:${EnvironmentParameter} + Image: + Fn::If: + - CreateDatabase + - Fn::Sub: ${EcrAccountNumberParameter}.dkr.ecr.${AWS::Region}.amazonaws.com/${AWS::StackName}/init-table:${EnvironmentParameter} + - Fn::Sub: ${EcrAccountNumberParameter}.dkr.ecr.${AWS::Region}.amazonaws.com/${MainCD2StackNameParameter}/init-table:${EnvironmentParameter} Essential: true LogConfiguration: LogDriver: awslogs @@ -880,7 +889,11 @@ Resources: - Name: !Sub ${AWS::StackName}-SyncTable Cpu: !Ref TaskCpuParameter Memory: !Ref TaskMemoryParameter - Image: !Sub ${EcrAccountNumberParameter}.dkr.ecr.${AWS::Region}.amazonaws.com/${AWS::StackName}/sync-table:${EnvironmentParameter} + Image: + Fn::If: + - CreateDatabase + - Fn::Sub: ${EcrAccountNumberParameter}.dkr.ecr.${AWS::Region}.amazonaws.com/${AWS::StackName}/sync-table:${EnvironmentParameter} + - Fn::Sub: ${EcrAccountNumberParameter}.dkr.ecr.${AWS::Region}.amazonaws.com/${MainCD2StackNameParameter}/sync-table:${EnvironmentParameter} Essential: true LogConfiguration: LogDriver: awslogs From 8ff9728bb462c4dd7e5ccbfd23317d4719ea88bd Mon Sep 17 00:00:00 2001 From: Sung Hwang Date: Thu, 14 Aug 2025 12:51:59 -0700 Subject: [PATCH 36/42] Added the steps to grant the necessary database permissions for the new cd2 database user. Also included the instruction for the additional cd2 stack. --- README.md | 10 ++++++++++ setup/prepare_aurora_db.py | 39 +++++++++++++++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f546703..7e08072 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,16 @@ pip install setup/requirements.txt -r ./setup/prepare_aurora_db.py --stack-name ``` +#### (Optional) If you are creating a new database user and schema for an additional Canvas instance: +If you create a new cloudformation stack for an additional Canvas instance, you need to modify `secret_name_prefix` so that it can target the correct secrets for the RDS database credential for the new DB user. + +When you run the `prepare_aurora_db.py` script, add `--is-additional-stack` argument. + +``` +pip install setup/requirements.txt -r +./setup/prepare_aurora_db.py --stack-name --is-additional-stack +``` + Occasionally the schema for a CD2 table will change. The DAP library will take care of applying these changes to the database, but they will not succeed if you have created views that depend on the table. To handle this situation, the `sync_table` Lambda function will attempt to drop and recreate any views that depend on the table being synced. The pgsql functions necessary to do this can be found in this repository: https://github.com/rvkulikov/pg-deps-management. You will need to run the `ddl.sql` script in your database to create the necessary functions. (details tbd) ## Configuration diff --git a/setup/prepare_aurora_db.py b/setup/prepare_aurora_db.py index 1c074b2..b5d5ebf 100644 --- a/setup/prepare_aurora_db.py +++ b/setup/prepare_aurora_db.py @@ -12,6 +12,12 @@ help="The name of the Canvas Data 2 CloudFormation stack containing the Aurora database", required=True, ) +parser.add_argument( + "--is-additional-stack", + help="Whether this is for the DB changes for the additional CD2 stack", + action="store_true", # If specified, sets the value as True + default=False # Default is False when not passed +) args = parser.parse_args() console = Console() @@ -22,6 +28,8 @@ cf_resource = boto3.resource("cloudformation") stack = cf_resource.Stack(args.stack_name) +is_additional_stack = args.is_additional_stack + console.print("Starting database preparation", style="bold green") # Fetch stack outputs and parameters @@ -127,6 +135,28 @@ def grant_user_to_admin(username, admin_username, database_name): except ClientError as e: console.print(f" ! Error granting user {username} to user {admin_username}: {e}", style="bold red") +def grant_create_permission_on_db_to_db_user(username, database_name): + """Grant CREATE permission on the database to the DB user""" + try: + grant_create_permission_sql = f"GRANT CREATE ON DATABASE {database_name} TO {username}" + execute_statement(grant_create_permission_sql, database_name) + console.print(f" - Granted CREATE permission on the database {database_name} to user {username}", style="bold green") + except ClientError as e: + console.print(f" ! Error granting CREATE permission on the database {database_name} to user {username}: {e}", style="bold red") + +def grant_access_permission_on_instructure_dap_schema_to_db_user(username, database_name): + """Grant SELECT, INSERT, UPDATE, DELETE permissions on the instructure_dap schema to the DB user""" + try: + tables = ["database_version", "table_sync"] + + for tablename in tables: + grant_access_permission_sql = f"GRANT INSERT, SELECT, UPDATE, DELETE ON instructure_dap.{tablename} TO {username}" + print(f"grant_access_permission_sql: {grant_access_permission_sql}") + execute_statement(grant_access_permission_sql, database_name) + console.print(f" - Granted SELECT, INSERT, UPDATE, and DELETE permission on the schema instructure_dap to user {username}", style="bold green") + except ClientError as e: + console.print(f" ! Error granting SELECT, INSERT, UPDATE, and DELETE permission on the schema instructure_dap to user {username}: {e}", style="bold red") + # Get all database user secrets secret_name_prefix = f"{prefix}-cd2-db-user-{env}-" user_secrets = secrets_client.list_secrets( @@ -163,4 +193,11 @@ def grant_user_to_admin(username, admin_username, database_name): assign_privileges(username, username, user_role, database_name) grant_usage_to_schema(username, "instructure_dap", database_name) - assign_privileges(username, "instructure_dap", user_role, database_name) \ No newline at end of file + assign_privileges(username, "instructure_dap", user_role, database_name) + + # Grant the CREATE privilege on the cd2 database. + grant_create_permission_on_db_to_db_user(username, database_name) + + # If this is for the new additional stack, grant the access permission to the instructure_dap schema. + if is_additional_stack: + grant_access_permission_on_instructure_dap_schema_to_db_user(username, database_name) \ No newline at end of file From f9e590e9e159268b92bcaab6f6aaa37fd42f1ee8 Mon Sep 17 00:00:00 2001 From: Sung Hwang Date: Thu, 14 Aug 2025 15:13:29 -0700 Subject: [PATCH 37/42] Included the ARN of the secrets for the catalog cd2 db user under the AthenaPostgreSQLConnectorExecutionPolicy. --- template.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/template.yaml b/template.yaml index e8cba56..5f80cce 100644 --- a/template.yaml +++ b/template.yaml @@ -1404,7 +1404,9 @@ Resources: - Effect: Allow Action: - secretsmanager:GetSecretValue - Resource: !Sub arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${ResourcePrefixParameter}-cd2-db-user-${EnvironmentParameter}-athena* + Resource: + - !Sub arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${ResourcePrefixParameter}-cd2-db-user-${EnvironmentParameter}-athena* + - !Sub arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${ResourcePrefixParameter}-catalog-cd2-db-user-${EnvironmentParameter}-athena* - Effect: Allow Action: - logs:CreateLogGroup From 8f5a8a34c22ccf1c5a4c12aba5ee710788c4c79d Mon Sep 17 00:00:00 2001 From: Sung Hwang Date: Thu, 14 Aug 2025 16:00:59 -0700 Subject: [PATCH 38/42] Renamed the secrets name for CD2 canvas/catalog user. Changed the ARNs for these secrets for AthenaPostgreSQLConnectorExecutionPolicy. --- template.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/template.yaml b/template.yaml index 5f80cce..3bab693 100644 --- a/template.yaml +++ b/template.yaml @@ -335,7 +335,7 @@ Resources: DatabaseUserSecretCanvas: Type: AWS::SecretsManager::Secret Properties: - Name: !Sub ${ResourcePrefixParameter}-${AWS::StackName}-cd2-db-user-${EnvironmentParameter}-${DatabaseCd2UserParameter} + Name: !Sub ${AWS::StackName}-cd2-db-user-${EnvironmentParameter}-${DatabaseCd2UserParameter} Description: Database user for Canvas Data 2 canvas user GenerateSecretString: SecretStringTemplate: !Sub '{"username": "${DatabaseCd2UserParameter}"}' @@ -1405,8 +1405,8 @@ Resources: Action: - secretsmanager:GetSecretValue Resource: - - !Sub arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${ResourcePrefixParameter}-cd2-db-user-${EnvironmentParameter}-athena* - - !Sub arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${ResourcePrefixParameter}-catalog-cd2-db-user-${EnvironmentParameter}-athena* + - !Sub arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${AWS::StackName}-cd2-db-user-${EnvironmentParameter}-athena* + - !Sub arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${AWS::StackName}-catalog-cd2-db-user-${EnvironmentParameter}-athena* - Effect: Allow Action: - logs:CreateLogGroup From e635e5fcf2f788f885cca74fba14709092b9b60e Mon Sep 17 00:00:00 2001 From: Sung Hwang Date: Thu, 14 Aug 2025 16:33:03 -0700 Subject: [PATCH 39/42] Reverted the secret name for AthenaPostgreSQLConnectorExecutionPolicy. --- template.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/template.yaml b/template.yaml index 3bab693..2b01211 100644 --- a/template.yaml +++ b/template.yaml @@ -1404,9 +1404,7 @@ Resources: - Effect: Allow Action: - secretsmanager:GetSecretValue - Resource: - - !Sub arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${AWS::StackName}-cd2-db-user-${EnvironmentParameter}-athena* - - !Sub arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${AWS::StackName}-catalog-cd2-db-user-${EnvironmentParameter}-athena* + Resource: !Sub arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:${ResourcePrefixParameter}-cd2-db-user-${EnvironmentParameter}-athena* - Effect: Allow Action: - logs:CreateLogGroup From 28ec3cc17d327553b8ad58055bf31d42e8807d89 Mon Sep 17 00:00:00 2001 From: Sung Hwang Date: Fri, 15 Aug 2025 10:11:19 -0700 Subject: [PATCH 40/42] Removed the stackname from the DefaultConnectionString from AthenaPostgreSQLConnector because it is the incorrect secrets name. --- template.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/template.yaml b/template.yaml index 2b01211..dd5fc3c 100644 --- a/template.yaml +++ b/template.yaml @@ -1343,7 +1343,7 @@ Resources: LambdaFunctionName: !Ref AthenaConnectorLambdaNameParameter CompositeHandler: PostGreSqlMuxCompositeHandler DefaultConnectionString: !Sub | - postgres://jdbc:postgresql://${AuroraDatabaseCluster.Endpoint.Address}:${AuroraDatabaseCluster.Endpoint.Port}/cd2?${!${ResourcePrefixParameter}-${AWS::StackName}-cd2-db-user-${EnvironmentParameter}-athena}&sslmode=verify-ca&sslfactory=org.postgresql.ssl.DefaultJavaSSLFactory&stringtype=unspecified + postgres://jdbc:postgresql://${AuroraDatabaseCluster.Endpoint.Address}:${AuroraDatabaseCluster.Endpoint.Port}/cd2?${!${ResourcePrefixParameter}-cd2-db-user-${EnvironmentParameter}-athena}&sslmode=verify-ca&sslfactory=org.postgresql.ssl.DefaultJavaSSLFactory&stringtype=unspecified DefaultScale: '0' DisableSpillEncryption: 'false' LambdaMemory: '3008' From b393ae9fee67185d35ecde9e7ecc7cd63c85bfc4 Mon Sep 17 00:00:00 2001 From: Sung Hwang Date: Mon, 25 Aug 2025 10:02:19 -0700 Subject: [PATCH 41/42] Removed the debug statement for a SQL query because it is considered a security risk. --- setup/prepare_aurora_db.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup/prepare_aurora_db.py b/setup/prepare_aurora_db.py index b5d5ebf..3813bcc 100644 --- a/setup/prepare_aurora_db.py +++ b/setup/prepare_aurora_db.py @@ -151,7 +151,6 @@ def grant_access_permission_on_instructure_dap_schema_to_db_user(username, datab for tablename in tables: grant_access_permission_sql = f"GRANT INSERT, SELECT, UPDATE, DELETE ON instructure_dap.{tablename} TO {username}" - print(f"grant_access_permission_sql: {grant_access_permission_sql}") execute_statement(grant_access_permission_sql, database_name) console.print(f" - Granted SELECT, INSERT, UPDATE, and DELETE permission on the schema instructure_dap to user {username}", style="bold green") except ClientError as e: From 6372fe18512fc6d6c0be98cbd9fe76dd887741aa Mon Sep 17 00:00:00 2001 From: Sung Hwang Date: Mon, 25 Aug 2025 10:05:11 -0700 Subject: [PATCH 42/42] Added the instruction on how how to deploy the CD2 cloudformation stack when you want to utilize the existing RDS database instance. --- README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7e08072..515856c 100644 --- a/README.md +++ b/README.md @@ -91,15 +91,21 @@ pip install setup/requirements.txt -r ``` #### (Optional) If you are creating a new database user and schema for an additional Canvas instance: -If you create a new cloudformation stack for an additional Canvas instance, you need to modify `secret_name_prefix` so that it can target the correct secrets for the RDS database credential for the new DB user. +1. If you create a new cloudformation stack for an additional Canvas instance, you need to modify `secret_name_prefix` so that it can target the correct secrets for the RDS database credential for the new DB user. -When you run the `prepare_aurora_db.py` script, add `--is-additional-stack` argument. +2. When you run the `prepare_aurora_db.py` script, add `--is-additional-stack` argument. ``` pip install setup/requirements.txt -r ./setup/prepare_aurora_db.py --stack-name --is-additional-stack ``` +3. You need to grant the access to the new schema for the database user `athena`. +- You need to run the following queries in order for the Athena connector to access all tables in the new schema: + - `GRANT SELECT ON ALL TABLES IN SCHEMA catalog TO athena;` + - `GRANT USAGE ON SCHEMA catalog TO athena;` + - `ALTER DEFAULT PRIVILEGES IN SCHEMA catalog GRANT SELECT ON TABLES TO athena;`: This allows any new tables created in catalog automatically give SELECT to the DB user `athena`. + Occasionally the schema for a CD2 table will change. The DAP library will take care of applying these changes to the database, but they will not succeed if you have created views that depend on the table. To handle this situation, the `sync_table` Lambda function will attempt to drop and recreate any views that depend on the table being synced. The pgsql functions necessary to do this can be found in this repository: https://github.com/rvkulikov/pg-deps-management. You will need to run the `ddl.sql` script in your database to create the necessary functions. (details tbd) ## Configuration