Skip to content

Commit

Permalink
Add docker-compose fixtures for S3 integration tests (#49107) (#49230)
Browse files Browse the repository at this point in the history 
Similarly to what has been done for Azure (#48636) and GCS (#48762),
this committ removes the existing Ant fixture that emulates a S3 storage
service in favor of multiple docker-compose based fixtures.

The goals here are multiple: be able to reuse a s3-fixture outside of the
repository-s3 plugin; allow parallel execution of integration tests; removes
the existing AmazonS3Fixture that has evolved in a weird beast in
dedicated, more maintainable fixtures.

The server side logic that emulates S3 mostly comes from the latest
HttpHandler made for S3 blob store repository tests, with additional
features extracted from the (now removed) AmazonS3Fixture:
authentication checks, session token checks and improved response
errors. Chunked upload request support for S3 object has been added
too.

The server side logic of all tests now reside in a single S3HttpHandler class.

Whereas AmazonS3Fixture contained logic for basic tests, session token
tests, EC2 tests or ECS tests, the S3 fixtures are now dedicated to each
kind of test. Fixtures are inheriting from each other, making things easier
to maintain.
  • Loading branch information
@tlrx
tlrx authored Nov 18, 2019
1 parent 8fd1eaa commit c5d7ad2
Show file tree
Hide file tree
Showing 12 changed files with 820 additions and 854 deletions.
94 changes: 32 additions & 62 deletions plugins/repository-s3/build.gradle
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import org.elasticsearch.gradle.MavenFilteringHack
import org.elasticsearch.gradle.test.AntFixture
import org.elasticsearch.gradle.test.RestIntegTestTask

import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
Expand All @@ -22,6 +21,7 @@ import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE
* specific language governing permissions and limitations
* under the License.
*/
apply plugin: 'elasticsearch.test.fixtures'

esplugin {
description 'The S3 repository plugin adds S3 repositories'
Expand All @@ -48,10 +48,10 @@ dependencies {
compile "joda-time:joda-time:${versions.joda}"

// HACK: javax.xml.bind was removed from default modules in java 9, so we pull the api in here,
// and whitelist this hack in JarHell
// and whitelist this hack in JarHell
compile 'javax.xml.bind:jaxb-api:2.2.2'

testCompile project(':test:fixtures:minio-fixture')
testCompile project(':test:fixtures:s3-fixture')
}

dependencyLicenses {
Expand Down Expand Up @@ -110,7 +110,7 @@ if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3P
s3PermanentAccessKey = 'access_key'
s3PermanentSecretKey = 'secret_key'
s3PermanentBucket = 'bucket'
s3PermanentBasePath = ''
s3PermanentBasePath = 'base_path'

useFixture = true

Expand All @@ -119,21 +119,21 @@ if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3P
}

if (!s3TemporaryAccessKey && !s3TemporarySecretKey && !s3TemporaryBucket && !s3TemporaryBasePath && !s3TemporarySessionToken) {
s3TemporaryAccessKey = 's3_integration_test_temporary_access_key'
s3TemporarySecretKey = 's3_integration_test_temporary_secret_key'
s3TemporaryBucket = 'temporary-bucket-test'
s3TemporaryBasePath = 'integration_test'
s3TemporarySessionToken = 's3_integration_test_temporary_session_token'
s3TemporaryAccessKey = 'session_token_access_key'
s3TemporarySecretKey = 'session_token_secret_key'
s3TemporaryBucket = 'session_token_bucket'
s3TemporaryBasePath = 'session_token_base_path'
s3TemporarySessionToken = 'session_token'

} else if (!s3TemporaryAccessKey || !s3TemporarySecretKey || !s3TemporaryBucket || !s3TemporaryBasePath || !s3TemporarySessionToken) {
throw new IllegalArgumentException("not all options specified to run against external S3 service as temporary credentials are present")
}

if (!s3EC2Bucket && !s3EC2BasePath && !s3ECSBucket && !s3ECSBasePath) {
s3EC2Bucket = 'ec2-bucket-test'
s3EC2BasePath = 'integration_test'
s3ECSBucket = 'ecs-bucket-test'
s3ECSBasePath = 'integration_test'
s3EC2Bucket = 'ec2_bucket'
s3EC2BasePath = 'ec2_base_path'
s3ECSBucket = 'ecs_bucket'
s3ECSBasePath = 'ecs_base_path'
} else if (!s3EC2Bucket || !s3EC2BasePath || !s3ECSBucket || !s3ECSBasePath) {
throw new IllegalArgumentException("not all options specified to run EC2/ECS tests are present")
}
Expand All @@ -147,8 +147,6 @@ task thirdPartyTest(type: Test) {
}

if (useFixture) {
apply plugin: 'elasticsearch.test.fixtures'

testFixtures.useFixture(':test:fixtures:minio-fixture')

def minioAddress = {
Expand Down Expand Up @@ -206,39 +204,6 @@ if (useFixture) {

check.dependsOn(thirdPartyTest)

File parentFixtures = new File(project.buildDir, "fixtures")
File s3FixtureFile = new File(parentFixtures, 's3Fixture.properties')

task s3FixtureProperties {
outputs.file(s3FixtureFile)
def s3FixtureOptions = [
"tests.seed" : project.testSeed,
"s3Fixture.permanent_bucket_name" : s3PermanentBucket,
"s3Fixture.permanent_key" : s3PermanentAccessKey,
"s3Fixture.temporary_bucket_name" : s3TemporaryBucket,
"s3Fixture.temporary_key" : s3TemporaryAccessKey,
"s3Fixture.temporary_session_token": s3TemporarySessionToken,
"s3Fixture.ec2_bucket_name" : s3EC2Bucket,
"s3Fixture.ecs_bucket_name" : s3ECSBucket,
"s3Fixture.disableChunkedEncoding" : s3DisableChunkedEncoding
]

doLast {
file(s3FixtureFile).text = s3FixtureOptions.collect { k, v -> "$k = $v" }.join("\n")
}
}

/** A task to start the AmazonS3Fixture which emulates an S3 service **/
task s3Fixture(type: AntFixture) {
dependsOn testClasses
dependsOn s3FixtureProperties
inputs.file(s3FixtureFile)

env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }"
executable = new File(project.runtimeJavaHome, 'bin/java')
args 'org.elasticsearch.repositories.s3.AmazonS3Fixture', baseDir, s3FixtureFile.getAbsolutePath()
}

processTestResources {
Map<String, Object> expansions = [
'permanent_bucket': s3PermanentBucket,
Expand All @@ -255,8 +220,13 @@ processTestResources {
MavenFilteringHack.filter(it, expansions)
}

integTest {
dependsOn s3Fixture
testFixtures.useFixture(':test:fixtures:s3-fixture')

def fixtureAddress = { fixture ->
assert useFixture: 'closure should not be used without a fixture'
int ephemeralPort = project(':test:fixtures:s3-fixture').postProcessFixture.ext."test.fixtures.${fixture}.tcp.80"
assert ephemeralPort > 0
'http://127.0.0.1:' + ephemeralPort
}

testClusters.integTest {
Expand All @@ -268,12 +238,12 @@ testClusters.integTest {
keystore 's3.client.integration_test_temporary.session_token', s3TemporarySessionToken

if (useFixture) {
setting 's3.client.integration_test_permanent.endpoint', { "http://${s3Fixture.addressAndPort}" }, IGNORE_VALUE
setting 's3.client.integration_test_temporary.endpoint', { "http://${s3Fixture.addressAndPort}" }, IGNORE_VALUE
setting 's3.client.integration_test_ec2.endpoint', { "http://${s3Fixture.addressAndPort}" }, IGNORE_VALUE
setting 's3.client.integration_test_permanent.endpoint', { "${-> fixtureAddress('s3-fixture')}" }, IGNORE_VALUE
setting 's3.client.integration_test_temporary.endpoint', { "${-> fixtureAddress('s3-fixture-with-session-token')}" }, IGNORE_VALUE
setting 's3.client.integration_test_ec2.endpoint', { "${-> fixtureAddress('s3-fixture-with-ec2')}" }, IGNORE_VALUE
// to redirect InstanceProfileCredentialsProvider to custom auth point
systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", { "http://${s3Fixture.addressAndPort}" }, IGNORE_VALUE
systemProperty "com.amazonaws.sdk.ec2MetadataServiceEndpointOverride", { "${-> fixtureAddress('s3-fixture-with-ec2')}" }, IGNORE_VALUE
} else {
println "Using an external service to test the repository-s3 plugin"
}
Expand All @@ -286,7 +256,7 @@ task s3ThirdPartyTests {
if (useFixture) {
task integTestECS(type: RestIntegTestTask.class) {
description = "Runs tests using the ECS repository."
dependsOn(project.s3Fixture)
dependsOn('bundlePlugin')
runner {
systemProperty 'tests.rest.blacklist', [
'repository_s3/10_basic/*',
Expand All @@ -299,9 +269,9 @@ if (useFixture) {
check.dependsOn(integTestECS)
testClusters.integTestECS {
setting 's3.client.integration_test_ecs.endpoint', { "http://${s3Fixture.addressAndPort}" }, IGNORE_VALUE
setting 's3.client.integration_test_ecs.endpoint', { "${-> fixtureAddress('s3-fixture-with-ecs')}" }, IGNORE_VALUE
plugin file(tasks.bundlePlugin.archiveFile)
environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI', { "http://${s3Fixture.addressAndPort}/ecs_credentials_endpoint" }, IGNORE_VALUE
environment 'AWS_CONTAINER_CREDENTIALS_FULL_URI', { "${-> fixtureAddress('s3-fixture-with-ecs')}/ecs_credentials_endpoint" }, IGNORE_VALUE
}
gradle.taskGraph.whenReady {
Expand All @@ -314,10 +284,10 @@ if (useFixture) {
thirdPartyAudit.ignoreMissingClasses (
// classes are missing
'javax.servlet.ServletContextEvent',
'javax.servlet.ServletContextListener',
'org.apache.avalon.framework.logger.Logger',
'org.apache.log.Hierarchy',
'javax.servlet.ServletContextEvent',
'javax.servlet.ServletContextListener',
'org.apache.avalon.framework.logger.Logger',
'org.apache.log.Hierarchy',
'org.apache.log.Logger',
'software.amazon.ion.IonReader',
'software.amazon.ion.IonSystem',
Expand All @@ -328,7 +298,7 @@ thirdPartyAudit.ignoreMissingClasses (
'software.amazon.ion.system.IonSystemBuilder',
'software.amazon.ion.system.IonTextWriterBuilder',
'software.amazon.ion.system.IonWriterBuilder',
// We don't use the kms dependency
// We don't use the kms dependency
'com.amazonaws.services.kms.AWSKMS',
'com.amazonaws.services.kms.AWSKMSClient',
'com.amazonaws.services.kms.model.DecryptRequest',
Expand Down
Loading

0 comments on commit c5d7ad2

Please sign in to comment.