diff --git a/.github/scripts/runCodeBuildBatchJob.sh b/.github/scripts/runCodeBuildBatchJob.sh new file mode 100755 index 00000000000..f18b15fb2f0 --- /dev/null +++ b/.github/scripts/runCodeBuildBatchJob.sh @@ -0,0 +1,63 @@ +#!/bin/bash + +error_exit() { + echo "$1" 1>&2 + exit 1 +} + +PROJECT_NAME=$1 +# get the source version to be built (defaults to main branch if not specified) +SOURCE_VERSION=${2:-main} + +echo "Starting CodeBuild project ${PROJECT_NAME}" + +# dump all GITHUB_* environment variables to file and pass to start job +jq -n 'env | to_entries | [.[] | select(.key | startswith("GITHUB_"))] | [.[] | {name: .key, value:.value, type:"PLAINTEXT"}]' >/tmp/gh_env_vars.json + +START_RESULT=$( + aws codebuild start-build-batch \ + --project-name ${PROJECT_NAME} \ + --source-version $SOURCE_VERSION \ + --environment-variables-override file:///tmp/gh_env_vars.json +) + +if [ "$?" != "0" ]; then + echo "Start result was: $START_RESULT" + error_exit "Could not start project. Exiting." +else + echo "Build started successfully." +fi + +BUILD_ID=$(echo ${START_RESULT} | jq '.buildBatch.id' -r) +echo "Build id $BUILD_ID" + +BUILD_STATUS="IN_PROGRESS" +while [ "$BUILD_STATUS" == "IN_PROGRESS" ]; do + echo "Checking build status." + BUILD=$(aws codebuild batch-get-build-batches --ids ${BUILD_ID}) + BUILD_STATUS=$(echo $BUILD | jq '.buildBatches[0].buildBatchStatus' -r) + + JOBS=$(echo $BUILD | jq '.buildBatches[0].buildGroups | [.[] | {identifier: .identifier, status: .currentBuildSummary.buildStatus} | select(.identifier | startswith("JOB"))]') + TOTAL_JOBS=$(echo $JOBS | jq 'length') + + SUCCEEDED_CNT=$(echo $JOBS | jq '[.[] | select(.status == "SUCCEEDED")] | length') + IN_PROGRESS_CNT=$(echo $JOBS | jq '[.[] | select(.status == "IN_PROGRESS")] | length') + + FAILED_CNT=$(($TOTAL_JOBS - $SUCCEEDED_CNT - $IN_PROGRESS_CNT)) + + if [ "$BUILD_STATUS" == "IN_PROGRESS" ]; then + echo "Build is still in progress (failed=$FAILED_CNT; in_progress=$IN_PROGRESS_CNT; succeeded=$SUCCEEDED_CNT; total=$TOTAL_JOBS), waiting..." + + fi + sleep 10 +done + +if [ "$BUILD_STATUS" != "SUCCEEDED" ]; then + BUILD=$(aws codebuild batch-get-build-batches --ids ${BUILD_ID}) + FAILED_BUILDS=$(echo $BUILD | jq '.buildBatches[0].buildGroups | [.[] | {identifier: .identifier, status: .currentBuildSummary.buildStatus} | select(.status == "FAILED")]') + echo "Failed builds in batch" + echo $FAILED_BUILDS + error_exit "Build failed, please review job output" +else + echo "Build succeeded." +fi diff --git a/.github/workflows/codebuild-ci.yml b/.github/workflows/codebuild-ci.yml new file mode 100644 index 00000000000..c1f4d7306d7 --- /dev/null +++ b/.github/workflows/codebuild-ci.yml @@ -0,0 +1,38 @@ +name: AWS CodeBuild CI + +on: + pull_request: + branches: + - main + workflow_dispatch: + +permissions: + id-token: write + +jobs: + e2e-tests: + runs-on: ubuntu-latest + steps: + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }} + aws-region: us-west-2 + - name: Run E2E Tests + uses: aws-actions/aws-codebuild-run-build@v1 + with: + project-name: gh-aws-sdk-kotlin-e2e-tests + + service-check-batch: + runs-on: ubuntu-latest + steps: + - name: Checkout sources + uses: actions/checkout@v2 + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }} + aws-region: us-west-2 + - name: Run Service Check Batch + run: | + .github/scripts/runCodeBuildBatchJob.sh gh-aws-sdk-kotlin-svc-check-batch ${{ github.event.pull_request.head.sha }} \ No newline at end of file diff --git a/services/s3/e2eTest/src/S3IntegrationTest.kt b/services/s3/e2eTest/src/S3IntegrationTest.kt index dd9956c13e2..7d852b46dca 100644 --- a/services/s3/e2eTest/src/S3IntegrationTest.kt +++ b/services/s3/e2eTest/src/S3IntegrationTest.kt @@ -35,12 +35,8 @@ import kotlin.time.ExperimentalTime @OptIn(ExperimentalTime::class, ExperimentalCoroutinesApi::class) @TestInstance(TestInstance.Lifecycle.PER_CLASS) class S3BucketOpsIntegrationTest { - companion object { - const val DEFAULT_REGION = "us-east-2" - } - private val client = S3Client { - region = DEFAULT_REGION + region = S3TestUtils.DEFAULT_REGION } private lateinit var testBucket: String @@ -216,58 +212,56 @@ class S3BucketOpsIntegrationTest { @Test fun testSelectObjectEventStream(): Unit = runBlocking { - S3Client.fromEnvironment().use { s3 -> - // upload our content to select from - val objKey = "developers.csv" - - val content = """ - Name,PhoneNumber,City,Occupation - Sam,(949) 555-6701,Irvine,Solutions Architect - Vinod,(949) 555-6702,Los Angeles,Solutions Architect - Jeff,(949) 555-6703,Seattle,AWS Evangelist - Jane,(949) 555-6704,Chicago,Developer - Sean,(949) 555-6705,Indianapolis,Developer - Mary,(949) 555-6706,Detroit,Developer - Kate,(949) 555-6707,Boston,Solutions Architect - """.trimIndent() + // upload our content to select from + val objKey = "developers.csv" + + val content = """ + Name,PhoneNumber,City,Occupation + Sam,(949) 555-6701,Irvine,Solutions Architect + Vinod,(949) 555-6702,Los Angeles,Solutions Architect + Jeff,(949) 555-6703,Seattle,AWS Evangelist + Jane,(949) 555-6704,Chicago,Developer + Sean,(949) 555-6705,Indianapolis,Developer + Mary,(949) 555-6706,Detroit,Developer + Kate,(949) 555-6707,Boston,Solutions Architect + """.trimIndent() - s3.putObject { - bucket = testBucket - key = objKey - body = ByteStream.fromString(content) - } + client.putObject { + bucket = testBucket + key = objKey + body = ByteStream.fromString(content) + } - // select content as an event stream - val req = SelectObjectContentRequest { - bucket = testBucket - key = objKey - expressionType = ExpressionType.Sql - expression = """SELECT * FROM s3object s where s."Name" = 'Jane'""" - inputSerialization { - csv { - fileHeaderInfo = FileHeaderInfo.Use - } - compressionType = CompressionType.None - } - outputSerialization { - csv { } + // select content as an event stream + val req = SelectObjectContentRequest { + bucket = testBucket + key = objKey + expressionType = ExpressionType.Sql + expression = """SELECT * FROM s3object s where s."Name" = 'Jane'""" + inputSerialization { + csv { + fileHeaderInfo = FileHeaderInfo.Use } + compressionType = CompressionType.None } - - val events = s3.selectObjectContent(req) { resp -> - // collect flow to list - resp.payload!!.toList() + outputSerialization { + csv { } } + } - assertEquals(3, events.size) + val events = client.selectObjectContent(req) { resp -> + // collect flow to list + resp.payload!!.toList() + } - val records = assertIs(events[0]) - assertIs(events[1]) - assertIs(events[2]) + assertEquals(3, events.size) - val expectedRecord = "Jane,(949) 555-6704,Chicago,Developer\n" - assertEquals(expectedRecord, records.value.payload?.decodeToString()) - } + val records = assertIs(events[0]) + assertIs(events[1]) + assertIs(events[2]) + + val expectedRecord = "Jane,(949) 555-6704,Chicago,Developer\n" + assertEquals(expectedRecord, records.value.payload?.decodeToString()) } } @@ -280,7 +274,7 @@ private fun File.chunk(partSize: Int): Sequence = internal suspend fun s3WithAllEngines(block: suspend (S3Client) -> Unit) { withAllEngines { engine -> S3Client { - region = S3BucketOpsIntegrationTest.DEFAULT_REGION + region = S3TestUtils.DEFAULT_REGION httpClientEngine = engine }.use { try { diff --git a/services/s3/e2eTest/src/S3PresignerTest.kt b/services/s3/e2eTest/src/S3PresignerTest.kt index 2a39d40163c..d3c386077e9 100644 --- a/services/s3/e2eTest/src/S3PresignerTest.kt +++ b/services/s3/e2eTest/src/S3PresignerTest.kt @@ -21,12 +21,9 @@ import kotlin.time.Duration.Companion.seconds @OptIn(ExperimentalCoroutinesApi::class) @TestInstance(TestInstance.Lifecycle.PER_CLASS) class S3PresignerTest { - companion object { - const val DEFAULT_REGION = "us-east-2" - } private val client = S3Client { - region = DEFAULT_REGION + region = S3TestUtils.DEFAULT_REGION } private lateinit var testBucket: String diff --git a/services/s3/e2eTest/src/S3TestUtils.kt b/services/s3/e2eTest/src/S3TestUtils.kt index e8fb3504896..7308c615c86 100644 --- a/services/s3/e2eTest/src/S3TestUtils.kt +++ b/services/s3/e2eTest/src/S3TestUtils.kt @@ -26,6 +26,8 @@ import kotlin.time.Duration.Companion.seconds object S3TestUtils { + const val DEFAULT_REGION = "us-west-2" + private const val TEST_BUCKET_PREFIX = "s3-test-bucket-" suspend fun getTestBucket(client: S3Client): String = getBucketWithPrefix(client, TEST_BUCKET_PREFIX) diff --git a/services/sts/e2eTest/src/STSPresignerTest.kt b/services/sts/e2eTest/src/STSPresignerTest.kt index 77823136538..88218acd104 100644 --- a/services/sts/e2eTest/src/STSPresignerTest.kt +++ b/services/sts/e2eTest/src/STSPresignerTest.kt @@ -21,7 +21,7 @@ class StsPresignerTest { @Test fun testGetCallerIdentityPresigner() = runBlocking { - val c = StsClient { region = "us-east-2" } + val c = StsClient { region = "us-west-2" } val req = GetCallerIdentityRequest { } val presignedRequest = req.presign(c.config, 60.seconds) diff --git a/services/transcribestreaming/e2eTest/src/TranscribeStreamingIntegrationTest.kt b/services/transcribestreaming/e2eTest/src/TranscribeStreamingIntegrationTest.kt index fed413138b0..987ebb48174 100644 --- a/services/transcribestreaming/e2eTest/src/TranscribeStreamingIntegrationTest.kt +++ b/services/transcribestreaming/e2eTest/src/TranscribeStreamingIntegrationTest.kt @@ -26,7 +26,7 @@ class TranscribeStreamingIntegrationTest { val url = this::class.java.classLoader.getResource("hello-kotlin-8000.wav") ?: error("failed to load test resource") val audioFile = Paths.get(url.toURI()).toFile() - TranscribeStreamingClient { region = "us-east-2" }.use { client -> + TranscribeStreamingClient { region = "us-west-2" }.use { client -> val transcript = getTranscript(client, audioFile) assertTrue(transcript.startsWith("Hello from", true), "full transcript: $transcript") }