Skip to content

chore(ci): add GH workflow for service check batch and e2e tests #713

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Oct 4, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 63 additions & 0 deletions .github/scripts/runCodeBuildBatchJob.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
#!/bin/bash

error_exit() {
echo "$1" 1>&2
exit 1
}

PROJECT_NAME=$1
# get the source version to be built (defaults to main branch if not specified)
SOURCE_VERSION=${2:-main}

echo "Starting CodeBuild project ${PROJECT_NAME}"

# dump all GITHUB_* environment variables to file and pass to start job
jq -n 'env | to_entries | [.[] | select(.key | startswith("GITHUB_"))] | [.[] | {name: .key, value:.value, type:"PLAINTEXT"}]' >/tmp/gh_env_vars.json

START_RESULT=$(
aws codebuild start-build-batch \
--project-name ${PROJECT_NAME} \
--source-version $SOURCE_VERSION \
--environment-variables-override file:///tmp/gh_env_vars.json
)

if [ "$?" != "0" ]; then
echo "Start result was: $START_RESULT"
error_exit "Could not start project. Exiting."
else
echo "Build started successfully."
fi

BUILD_ID=$(echo ${START_RESULT} | jq '.buildBatch.id' -r)
echo "Build id $BUILD_ID"

BUILD_STATUS="IN_PROGRESS"
while [ "$BUILD_STATUS" == "IN_PROGRESS" ]; do
echo "Checking build status."
BUILD=$(aws codebuild batch-get-build-batches --ids ${BUILD_ID})
BUILD_STATUS=$(echo $BUILD | jq '.buildBatches[0].buildBatchStatus' -r)

JOBS=$(echo $BUILD | jq '.buildBatches[0].buildGroups | [.[] | {identifier: .identifier, status: .currentBuildSummary.buildStatus} | select(.identifier | startswith("JOB"))]')
TOTAL_JOBS=$(echo $JOBS | jq 'length')

SUCCEEDED_CNT=$(echo $JOBS | jq '[.[] | select(.status == "SUCCEEDED")] | length')
IN_PROGRESS_CNT=$(echo $JOBS | jq '[.[] | select(.status == "IN_PROGRESS")] | length')

FAILED_CNT=$(($TOTAL_JOBS - $SUCCEEDED_CNT - $IN_PROGRESS_CNT))

if [ "$BUILD_STATUS" == "IN_PROGRESS" ]; then
echo "Build is still in progress (failed=$FAILED_CNT; in_progress=$IN_PROGRESS_CNT; succeeded=$SUCCEEDED_CNT; total=$TOTAL_JOBS), waiting..."

fi
sleep 10
done

if [ "$BUILD_STATUS" != "SUCCEEDED" ]; then
BUILD=$(aws codebuild batch-get-build-batches --ids ${BUILD_ID})
FAILED_BUILDS=$(echo $BUILD | jq '.buildBatches[0].buildGroups | [.[] | {identifier: .identifier, status: .currentBuildSummary.buildStatus} | select(.status == "FAILED")]')
echo "Failed builds in batch"
echo $FAILED_BUILDS
error_exit "Build failed, please review job output"
else
echo "Build succeeded."
fi
38 changes: 38 additions & 0 deletions .github/workflows/codebuild-ci.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
name: AWS CodeBuild CI

on:
pull_request:
branches:
- main
workflow_dispatch:

permissions:
id-token: write

jobs:
e2e-tests:
runs-on: ubuntu-latest
steps:
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v1
with:
role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
aws-region: us-west-2
- name: Run E2E Tests
uses: aws-actions/aws-codebuild-run-build@v1
with:
project-name: gh-aws-sdk-kotlin-e2e-tests

service-check-batch:
runs-on: ubuntu-latest
steps:
- name: Checkout sources
uses: actions/checkout@v2
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v1
with:
role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
aws-region: us-west-2
- name: Run Service Check Batch
run: |
.github/scripts/runCodeBuildBatchJob.sh gh-aws-sdk-kotlin-svc-check-batch ${{ github.event.pull_request.head.sha }}
94 changes: 44 additions & 50 deletions services/s3/e2eTest/src/S3IntegrationTest.kt
Original file line number Diff line number Diff line change
Expand Up @@ -35,12 +35,8 @@ import kotlin.time.ExperimentalTime
@OptIn(ExperimentalTime::class, ExperimentalCoroutinesApi::class)
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class S3BucketOpsIntegrationTest {
companion object {
const val DEFAULT_REGION = "us-east-2"
}

private val client = S3Client {
region = DEFAULT_REGION
region = S3TestUtils.DEFAULT_REGION
}

private lateinit var testBucket: String
Expand Down Expand Up @@ -216,58 +212,56 @@ class S3BucketOpsIntegrationTest {

@Test
fun testSelectObjectEventStream(): Unit = runBlocking {
S3Client.fromEnvironment().use { s3 ->
// upload our content to select from
val objKey = "developers.csv"

val content = """
Name,PhoneNumber,City,Occupation
Sam,(949) 555-6701,Irvine,Solutions Architect
Vinod,(949) 555-6702,Los Angeles,Solutions Architect
Jeff,(949) 555-6703,Seattle,AWS Evangelist
Jane,(949) 555-6704,Chicago,Developer
Sean,(949) 555-6705,Indianapolis,Developer
Mary,(949) 555-6706,Detroit,Developer
Kate,(949) 555-6707,Boston,Solutions Architect
""".trimIndent()
// upload our content to select from
val objKey = "developers.csv"

val content = """
Name,PhoneNumber,City,Occupation
Sam,(949) 555-6701,Irvine,Solutions Architect
Vinod,(949) 555-6702,Los Angeles,Solutions Architect
Jeff,(949) 555-6703,Seattle,AWS Evangelist
Jane,(949) 555-6704,Chicago,Developer
Sean,(949) 555-6705,Indianapolis,Developer
Mary,(949) 555-6706,Detroit,Developer
Kate,(949) 555-6707,Boston,Solutions Architect
""".trimIndent()

s3.putObject {
bucket = testBucket
key = objKey
body = ByteStream.fromString(content)
}
client.putObject {
bucket = testBucket
key = objKey
body = ByteStream.fromString(content)
}

// select content as an event stream
val req = SelectObjectContentRequest {
bucket = testBucket
key = objKey
expressionType = ExpressionType.Sql
expression = """SELECT * FROM s3object s where s."Name" = 'Jane'"""
inputSerialization {
csv {
fileHeaderInfo = FileHeaderInfo.Use
}
compressionType = CompressionType.None
}
outputSerialization {
csv { }
// select content as an event stream
val req = SelectObjectContentRequest {
bucket = testBucket
key = objKey
expressionType = ExpressionType.Sql
expression = """SELECT * FROM s3object s where s."Name" = 'Jane'"""
inputSerialization {
csv {
fileHeaderInfo = FileHeaderInfo.Use
}
compressionType = CompressionType.None
}

val events = s3.selectObjectContent(req) { resp ->
// collect flow to list
resp.payload!!.toList()
outputSerialization {
csv { }
}
}

assertEquals(3, events.size)
val events = client.selectObjectContent(req) { resp ->
// collect flow to list
resp.payload!!.toList()
}

val records = assertIs<SelectObjectContentEventStream.Records>(events[0])
assertIs<SelectObjectContentEventStream.Stats>(events[1])
assertIs<SelectObjectContentEventStream.End>(events[2])
assertEquals(3, events.size)

val expectedRecord = "Jane,(949) 555-6704,Chicago,Developer\n"
assertEquals(expectedRecord, records.value.payload?.decodeToString())
}
val records = assertIs<SelectObjectContentEventStream.Records>(events[0])
assertIs<SelectObjectContentEventStream.Stats>(events[1])
assertIs<SelectObjectContentEventStream.End>(events[2])

val expectedRecord = "Jane,(949) 555-6704,Chicago,Developer\n"
assertEquals(expectedRecord, records.value.payload?.decodeToString())
}
}

Expand All @@ -280,7 +274,7 @@ private fun File.chunk(partSize: Int): Sequence<LongRange> =
internal suspend fun s3WithAllEngines(block: suspend (S3Client) -> Unit) {
withAllEngines { engine ->
S3Client {
region = S3BucketOpsIntegrationTest.DEFAULT_REGION
region = S3TestUtils.DEFAULT_REGION
httpClientEngine = engine
}.use {
try {
Expand Down
5 changes: 1 addition & 4 deletions services/s3/e2eTest/src/S3PresignerTest.kt
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,9 @@ import kotlin.time.Duration.Companion.seconds
@OptIn(ExperimentalCoroutinesApi::class)
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class S3PresignerTest {
companion object {
const val DEFAULT_REGION = "us-east-2"
}

private val client = S3Client {
region = DEFAULT_REGION
region = S3TestUtils.DEFAULT_REGION
}

private lateinit var testBucket: String
Expand Down
2 changes: 2 additions & 0 deletions services/s3/e2eTest/src/S3TestUtils.kt
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ import kotlin.time.Duration.Companion.seconds

object S3TestUtils {

const val DEFAULT_REGION = "us-west-2"

private const val TEST_BUCKET_PREFIX = "s3-test-bucket-"

suspend fun getTestBucket(client: S3Client): String = getBucketWithPrefix(client, TEST_BUCKET_PREFIX)
Expand Down
2 changes: 1 addition & 1 deletion services/sts/e2eTest/src/STSPresignerTest.kt
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ class StsPresignerTest {

@Test
fun testGetCallerIdentityPresigner() = runBlocking {
val c = StsClient { region = "us-east-2" }
val c = StsClient { region = "us-west-2" }
val req = GetCallerIdentityRequest { }
val presignedRequest = req.presign(c.config, 60.seconds)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ class TranscribeStreamingIntegrationTest {
val url = this::class.java.classLoader.getResource("hello-kotlin-8000.wav") ?: error("failed to load test resource")
val audioFile = Paths.get(url.toURI()).toFile()

TranscribeStreamingClient { region = "us-east-2" }.use { client ->
TranscribeStreamingClient { region = "us-west-2" }.use { client ->
val transcript = getTranscript(client, audioFile)
assertTrue(transcript.startsWith("Hello from", true), "full transcript: $transcript")
}
Expand Down