diff --git a/.buildkite/dra.sh b/.buildkite/dra.sh index 039aaf1e0..8f25e4a5f 100644 --- a/.buildkite/dra.sh +++ b/.buildkite/dra.sh @@ -38,7 +38,7 @@ mkdir localRepo wget --quiet "https://artifacts-$DRA_WORKFLOW.elastic.co/elasticsearch/${ES_BUILD_ID}/maven/org/elasticsearch/gradle/build-tools/${HADOOP_VERSION}${VERSION_SUFFIX}/build-tools-${HADOOP_VERSION}${VERSION_SUFFIX}.jar" \ -O "localRepo/build-tools-${HADOOP_VERSION}${VERSION_SUFFIX}.jar" -./gradlew -S -PlocalRepo=true "${BUILD_ARGS}" -Dorg.gradle.warning.mode=summary -Dcsv="$WORKSPACE/build/distributions/dependencies-${HADOOP_VERSION}${VERSION_SUFFIX}.csv" :dist:generateDependenciesReport distribution +./gradlew -S -PlocalRepo=true "${BUILD_ARGS}" -Dorg.gradle.warning.mode=summary -Dcsv="$WORKSPACE/build/distributions/dependencies-${HADOOP_VERSION}${VERSION_SUFFIX}.csv" :dist:generateDependenciesReport distribution zipAggregation # Allow other users access to read the artifacts so they are readable in the container find "$WORKSPACE" -type f -path "*/build/distributions/*" -exec chmod a+r {} \; diff --git a/build.gradle b/build.gradle index 4cad50b27..88a2ee63c 100644 --- a/build.gradle +++ b/build.gradle @@ -1,6 +1,7 @@ import org.elasticsearch.hadoop.gradle.buildtools.ConcatFilesTask import java.lang.management.ManagementFactory; import java.time.LocalDateTime; +import org.elasticsearch.gradle.VersionProperties import org.elasticsearch.gradle.Architecture import org.elasticsearch.gradle.OS @@ -11,9 +12,24 @@ import java.time.LocalDateTime description = 'Elasticsearch for Apache Hadoop' apply plugin: 'es.hadoop.build.root' +apply plugin: 'com.gradleup.nmcp.aggregation' defaultTasks 'build' +dependencies { + nmcpAggregation(project(":dist")) + nmcpAggregation(project(":elasticsearch-hadoop-mr")) + nmcpAggregation(project(":elasticsearch-hadoop-hive")) + nmcpAggregation(project(":elasticsearch-spark-20")) + nmcpAggregation(project(":elasticsearch-spark-30")) +} + +tasks.named('zipAggregation').configure { + archiveFileName.unset(); + archiveBaseName.set("elasticsearch-maven-aggregration") + archiveVersion.set(VersionProperties.elasticsearch) +} + allprojects { group = "org.elasticsearch" tasks.withType(AbstractCopyTask) { diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 9cf0984cd..f29b691c8 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -72,6 +72,7 @@ dependencies { // Required for dependency licenses task implementation 'org.apache.rat:apache-rat:0.11' implementation 'commons-codec:commons-codec:1.12' + implementation 'com.gradleup.nmcp:nmcp:0.1.4' if (localRepo) { implementation name: "build-tools-${buildToolsVersion}" diff --git a/buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy index 458d47a68..54e9c1e77 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy @@ -567,6 +567,7 @@ class BuildPlugin implements Plugin { private void configureMaven(Project project) { project.getPluginManager().apply("maven-publish") + project.getPluginManager().apply("com.gradleup.nmcp") // Configure Maven publication project.publishing { @@ -587,13 +588,6 @@ class BuildPlugin implements Plugin { // Configure Maven Pom configurePom(project, project.publishing.publications.main) - // Disable the publishing tasks since we only need the pom generation tasks. - // If we are working with a project that has a scala variant (see below), we need to modify the pom's - // artifact id which the publish task does not like (it fails validation when run). - project.getTasks().withType(PublishToMavenRepository) { PublishToMavenRepository m -> - m.enabled = false - } - // Configure Scala Variants if present project.getPlugins().withType(SparkVariantPlugin).whenPluginAdded { // Publishing gets weird when you introduce variants into the project. @@ -608,7 +602,7 @@ class BuildPlugin implements Plugin { // Main variant needs the least configuration on its own, since it is the default publication created above. sparkVariants.defaultVariant { SparkVariant variant -> - updateVariantPomLocationAndArtifactId(project, project.publishing.publications.main, variant) + updateVariantArtifactId(project, project.publishing.publications.main, variant) } // For each spark variant added, we need to do a few things: @@ -658,8 +652,9 @@ class BuildPlugin implements Plugin { from variantComponent suppressAllPomMetadataWarnings() // We get it. Gradle metadata is better than Maven Poms } + variantPublication.setAlias(true) configurePom(project, variantPublication) - updateVariantPomLocationAndArtifactId(project, variantPublication, variant) + updateVariantArtifactId(project, variantPublication, variant) } } } @@ -672,14 +667,6 @@ class BuildPlugin implements Plugin { } private static void configurePom(Project project, MavenPublication publication) { - // Set the pom's destination to the distribution directory - project.tasks.withType(GenerateMavenPom).all { GenerateMavenPom pom -> - if (pom.name == "generatePomFileFor${publication.name.capitalize()}Publication") { - BasePluginExtension baseExtension = project.getExtensions().getByType(BasePluginExtension.class); - pom.destination = project.provider({"${project.buildDir}/distributions/${baseExtension.archivesName.get()}-${project.getVersion()}.pom"}) - } - } - // add all items necessary for publication Provider descriptionProvider = project.provider({ project.getDescription() }) MavenPom pom = publication.getPom() @@ -722,7 +709,8 @@ class BuildPlugin implements Plugin { while (dependenciesIterator.hasNext()) { Node dependencyNode = dependenciesIterator.next() String artifact = dependencyNode.get("artifactId").text() - if (artifact == dependency.getName()) { + // handle scala variants by splitting via "_" and checking the first part + if (artifact =~ dependency.getName().split('_')[0]) { dependenciesIterator.remove() break } @@ -732,23 +720,11 @@ class BuildPlugin implements Plugin { } } - private static void updateVariantPomLocationAndArtifactId(Project project, MavenPublication publication, SparkVariant variant) { + private static void updateVariantArtifactId(Project project, MavenPublication publication, SparkVariant variant) { // Add variant classifier to the pom file name if required - String classifier = variant.shouldClassifySparkVersion() && variant.isDefaultVariant() == false ? "-${variant.getName()}" : '' BasePluginExtension baseExtension = project.getExtensions().getByType(BasePluginExtension.class); - String filename = "${baseExtension.archivesName.get()}_${variant.scalaMajorVersion}-${project.getVersion()}${classifier}" - // Fix the pom name - project.tasks.withType(GenerateMavenPom).all { GenerateMavenPom pom -> - if (pom.name == "generatePomFileFor${publication.name.capitalize()}Publication") { - pom.destination = project.provider({"${project.buildDir}/distributions/${filename}.pom"}) - } - } - // Fix the artifactId. Note: The publishing task does not like this happening. Hence it is disabled. - publication.getPom().withXml { XmlProvider xml -> - Node root = xml.asNode() - Node artifactId = (root.get('artifactId') as NodeList).get(0) as Node - artifactId.setValue("${baseExtension.archivesName.get()}_${variant.scalaMajorVersion}") - } + // Fix the artifact id + publication.setArtifactId("${baseExtension.archivesName.get()}_${variant.scalaMajorVersion}") } /** diff --git a/dist/build.gradle b/dist/build.gradle index 5340158e7..326b2a7f5 100644 --- a/dist/build.gradle +++ b/dist/build.gradle @@ -132,26 +132,6 @@ javadoc { } } -publishing { - publications { - main { - getPom().withXml { XmlProvider xml -> - Node root = xml.asNode() - - // add clojars repo to pom - Node repositories = root.appendNode('repositories') - Node repository = repositories.appendNode('repository') - repository.appendNode('id', 'clojars.org') - repository.appendNode('url', 'https://clojars.org/repo') - BasePluginExtension baseExtension = project.getExtensions().getByType(BasePluginExtension.class) - - // Correct the artifact Id, otherwise it is listed as 'dist' - root.get('artifactId').get(0).setValue(baseExtension.archivesName.get()) - } - } - } -} - // Name of the directory under the root of the zip file that will contain the zip contents String zipContentDir = "elasticsearch-hadoop-${project.version}" @@ -179,8 +159,28 @@ task('distZip', type: Zip) { distribution { dependsOn(distZip) + +} + + +publishing { + publications { + main { + artifact tasks.named('distZip') + getPom().withXml { XmlProvider xml -> + Node root = xml.asNode() + + // add clojars repo to pom + Node repositories = root.appendNode('repositories') + Node repository = repositories.appendNode('repository') + repository.appendNode('id', 'clojars.org') + repository.appendNode('url', 'https://clojars.org/repo') + } + } + } } + // Add a task in the root project that collects all the dependencyReport data for each project // Concatenates the dependencies CSV files into a single file task generateDependenciesReport(type: ConcatFilesTask) { concatDepsTask -> @@ -197,3 +197,15 @@ task generateDependenciesReport(type: ConcatFilesTask) { concatDepsTask -> project.tasks.named('dependencyLicenses', DependencyLicensesTask) { it.dependencies = project.configurations.licenseChecks } + + +tasks.register('copyPoms', Copy) { + BasePluginExtension baseExtension = project.getExtensions().getByType(BasePluginExtension.class); + from(tasks.named('generatePomFileForMainPublication')) + into(new File(project.buildDir, 'distributions')) + rename 'pom-default.xml', "${baseExtension.archivesName.get()}-${project.getVersion()}.pom" +} + +tasks.named('distribution').configure { + dependsOn 'copyPoms' +} \ No newline at end of file diff --git a/hive/build.gradle b/hive/build.gradle index 737384582..15465e936 100644 --- a/hive/build.gradle +++ b/hive/build.gradle @@ -88,3 +88,15 @@ itestJar { include "META-INF/services/*" } } + + +tasks.register('copyPoms', Copy) { + BasePluginExtension baseExtension = project.getExtensions().getByType(BasePluginExtension.class); + from(tasks.named('generatePomFileForMainPublication')) + into(new File(project.buildDir, 'distributions')) + rename 'pom-default.xml', "${baseExtension.archivesName.get()}-${project.getVersion()}.pom" +} + +tasks.named('distribution').configure { + dependsOn 'copyPoms' +} diff --git a/mr/build.gradle b/mr/build.gradle index cceb5d721..5e2d9569f 100644 --- a/mr/build.gradle +++ b/mr/build.gradle @@ -137,3 +137,14 @@ eclipse.classpath.file { } } } + +tasks.register('copyPoms', Copy) { + BasePluginExtension baseExtension = project.getExtensions().getByType(BasePluginExtension.class); + from(tasks.named('generatePomFileForMainPublication')) + into(new File(project.buildDir, 'distributions')) + rename 'pom-default.xml', "${baseExtension.archivesName.get()}-${project.getVersion()}.pom" +} + +tasks.named('distribution').configure { + dependsOn 'copyPoms' +} diff --git a/spark/sql-20/build.gradle b/spark/sql-20/build.gradle index 036180e5a..ddfb3acd6 100644 --- a/spark/sql-20/build.gradle +++ b/spark/sql-20/build.gradle @@ -217,3 +217,17 @@ sparkVariants { } } } + +tasks.register('copyPoms', Copy) { + from(tasks.named('generatePomFileForMainPublication')) { + rename 'pom-default.xml', "elasticsearch-spark-20_2.12-${project.getVersion()}.pom" + } + from(tasks.named('generatePomFileForSpark20scala211Publication')) { + rename 'pom-default.xml', "elasticsearch-spark-20_2.11-${project.getVersion()}.pom" + } + into(new File(project.buildDir, 'distributions')) +} + +tasks.named('distribution').configure { + dependsOn 'copyPoms' +} diff --git a/spark/sql-30/build.gradle b/spark/sql-30/build.gradle index 45665d4f7..db49af8c7 100644 --- a/spark/sql-30/build.gradle +++ b/spark/sql-30/build.gradle @@ -189,3 +189,18 @@ sparkVariants { } } } + + +tasks.register('copyPoms', Copy) { + from(tasks.named('generatePomFileForMainPublication')) { + rename 'pom-default.xml', "elasticsearch-spark-30_2.12-${project.getVersion()}.pom" + } + from(tasks.named('generatePomFileForSpark30scala213Publication')) { + rename 'pom-default.xml', "elasticsearch-spark-30_2.13-${project.getVersion()}.pom" + } + into(new File(project.buildDir, 'distributions')) +} + +tasks.named('distribution').configure { + dependsOn 'copyPoms' +}