From 5df4dc122ef80a669b4de9bf60a25645cc0b0b87 Mon Sep 17 00:00:00 2001
From: Luke Cwik <lcwik@google.com>
Date: Mon, 9 Apr 2018 09:26:11 -0700
Subject: [PATCH] [BEAM-4014] Replace the Java maven based PostCommit with a
Gradle based version.
https://builds.apache.org/view/A-D/view/Beam/job/beam_PreCommit_Java_MavenInstall/19300/testReport/api/json
shows the list of ITs that were executed. Note that its all the examples/java ITs and the sdks/java/io/google-cloud-platform ITs.
---
...b_beam_PostCommit_Java_GradleBuild.groovy} | 58 ++++++++++---------
...job_beam_PreCommit_Java_GradleBuild.groovy | 2 +-
build.gradle | 7 ++-
.../google-cloud-dataflow-java/build.gradle | 52 ++++++++++++++++-
.../io/google-cloud-platform/build.gradle | 36 +++++++++---
5 files changed, 118 insertions(+), 37 deletions(-)
rename .test-infra/jenkins/{job_beam_PostCommit_Java_MavenInstall.groovy => job_beam_PostCommit_Java_GradleBuild.groovy} (55%)
diff --git a/.test-infra/jenkins/job_beam_PostCommit_Java_MavenInstall.groovy b/.test-infra/jenkins/job_beam_PostCommit_Java_GradleBuild.groovy
similarity index 55%
rename from .test-infra/jenkins/job_beam_PostCommit_Java_MavenInstall.groovy
rename to .test-infra/jenkins/job_beam_PostCommit_Java_GradleBuild.groovy
index 0dda772a230..ba0da5dab44 100644
--- a/.test-infra/jenkins/job_beam_PostCommit_Java_MavenInstall.groovy
+++ b/.test-infra/jenkins/job_beam_PostCommit_Java_GradleBuild.groovy
@@ -20,10 +20,10 @@ import common_job_properties
// This job runs the Java postcommit tests, including the suite of integration
// tests.
-mavenJob('beam_PostCommit_Java_MavenInstall') {
- description('Runs postcommit tests on the Java SDK.')
-
+job('beam_PostCommit_Java_GradleBuild') {
+ description('Runs PostCommit tests on the Java SDK.')
previousNames('beam_PostCommit_MavenVerify')
+ previousNames('beam_PostCommit_Java_MavenInstall')
// Execute concurrent builds if necessary.
concurrentBuild()
@@ -31,35 +31,39 @@ mavenJob('beam_PostCommit_Java_MavenInstall') {
// Set common parameters.
common_job_properties.setTopLevelMainJobProperties(delegate, 'master', 240)
- // Set maven parameters.
- common_job_properties.setMavenConfig(delegate)
+ def gradle_switches = [
+ // Gradle log verbosity enough to diagnose basic build issues
+ "--info",
+ // Continue the build even if there is a failure to show as many potential failures as possible.
+ '--continue',
+ // Until we verify the build cache is working appropriately, force rerunning all tasks
+ '--rerun-tasks',
+ // Specify maven home on Jenkins, needed by Maven archetype integration tests.
+ '-Pmaven_home=/home/jenkins/tools/maven/apache-maven-3.5.2'
+ ]
+
+ // Publish all test results to Jenkins
+ publishers {
+ archiveJunit('**/build/test-results/**/*.xml')
+ }
// Sets that this is a PostCommit job.
common_job_properties.setPostCommit(delegate)
// Allows triggering this build against pull requests.
common_job_properties.enablePhraseTriggeringFromPullRequest(
- delegate,
- 'Java SDK Post Commit Tests',
- 'Run Java PostCommit')
+ delegate,
+ 'Java SDK Post Commit Tests',
+ 'Run Java PostCommit')
- // Maven goals for this job.
- goals([
- 'clean',
- 'install',
- '--projects sdks/java/core,runners/direct-java,sdks/java/fn-execution',
- ' --also-make',
- '--also-make-dependents',
- '--batch-mode',
- '--errors',
- '--fail-at-end',
- '-P release,dataflow-runner',
- '-DrepoToken=$COVERALLS_REPO_TOKEN',
- '-D skipITs=false',
- '''-D integrationTestPipelineOptions=\'[ \
- "--project=apache-beam-testing", \
- "--tempRoot=gs://temp-storage-for-end-to-end-tests", \
- "--runner=TestDataflowRunner" \
- ]\' '''
- ].join(' '))
+ // Gradle goals for this job.
+ steps {
+ gradle {
+ rootBuildScriptDir(common_job_properties.checkoutDir)
+ tasks(':javaPostCommit')
+ for (String gradle_switch : gradle_switches) {
+ switches(gradle_switch)
+ }
+ }
+ }
}
diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_GradleBuild.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_GradleBuild.groovy
index fe3e1ac76c6..66216708546 100644
--- a/.test-infra/jenkins/job_beam_PreCommit_Java_GradleBuild.groovy
+++ b/.test-infra/jenkins/job_beam_PreCommit_Java_GradleBuild.groovy
@@ -44,7 +44,7 @@ job('beam_PreCommit_Java_GradleBuild') {
'--continue',
// Until we verify the build cache is working appropriately, force rerunning all tasks
'--rerun-tasks',
- // Specify maven home on Jenkins
+ // Specify maven home on Jenkins, needed by Maven archetype integration tests.
'-Pmaven_home=/home/jenkins/tools/maven/apache-maven-3.5.2'
]
diff --git a/build.gradle b/build.gradle
index d4ebafe30e7..102fffbe687 100644
--- a/build.gradle
+++ b/build.gradle
@@ -109,7 +109,7 @@ rat {
}
check.dependsOn rat
-// Define root PreCommit tasks simplifying what is needed
+// Define root pre/post commit tasks simplifying what is needed
// to be specified on the commandline when executing locally.
// This indirection also makes Jenkins use the branch of the PR
// for the test definitions.
@@ -122,6 +122,11 @@ task javaPreCommit() {
dependsOn ":beam-examples-java:preCommit"
}
+task javaPostCommit() {
+ dependsOn ":javaPreCommit"
+ dependsOn ":beam-runners-google-cloud-dataflow-java:postCommit"
+}
+
task goPreCommit() {
dependsOn ":rat"
dependsOn ":beam-sdks-go:test"
diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle
index 555a61b1e63..cb3c6a42582 100644
--- a/runners/google-cloud-dataflow-java/build.gradle
+++ b/runners/google-cloud-dataflow-java/build.gradle
@@ -51,6 +51,8 @@ test {
configurations {
validatesRunner
+ examplesJavaIntegrationTest
+ googleCloudPlatformIntegrationTest
}
dependencies {
@@ -89,6 +91,10 @@ dependencies {
shadowTest library.java.jackson_dataformat_yaml
validatesRunner project(path: ":beam-sdks-java-core", configuration: "shadowTest")
validatesRunner project(path: project.path, configuration: "shadow")
+ examplesJavaIntegrationTest project(path: project.path, configuration: "shadow")
+ examplesJavaIntegrationTest project(path: ":beam-examples-java", configuration: "shadowTest")
+ googleCloudPlatformIntegrationTest project(path: project.path, configuration: "shadow")
+ googleCloudPlatformIntegrationTest project(path: ":beam-sdks-java-io-google-cloud-platform", configuration: "shadowTest")
}
test {
@@ -120,7 +126,6 @@ task validatesRunnerTest(type: Test) {
excludeCategories 'org.apache.beam.sdk.testing.UsesUnboundedPCollections'
excludeCategories 'org.apache.beam.sdk.testing.UsesTestStream'
excludeCategories 'org.apache.beam.sdk.testing.UsesParDoLifecycle'
-
}
}
@@ -134,3 +139,48 @@ task validatesRunner {
def gcpProject = project.findProperty('gcpProject') ?: 'apache-beam-testing'
def gcsBucket = project.findProperty('gcsBucket') ?: 'temp-storage-for-release-validation-tests/quickstart'
createJavaQuickstartValidationTask(name: 'Dataflow', gcpProject: gcpProject, gcsBucket: gcsBucket)
+
+task googleCloudPlatformIntegrationTest(type: Test) {
+ group = "Verification"
+ def dataflowProject = project.findProperty('dataflowProject') ?: 'apache-beam-testing'
+ def dataflowTempRoot = project.findProperty('dataflowTempRoot') ?: 'gs://temp-storage-for-end-to-end-tests'
+ systemProperty "beamTestPipelineOptions", JsonOutput.toJson([
+ "--runner=TestDataflowRunner",
+ "--project=${dataflowProject}",
+ "--tempRoot=${dataflowTempRoot}",
+ ])
+
+ include '**/*IT.class'
+ maxParallelForks 4
+ classpath = configurations.googleCloudPlatformIntegrationTest
+ testClassesDirs = files(project(":beam-sdks-java-io-google-cloud-platform").sourceSets.test.output.classesDirs)
+ useJUnit { }
+}
+
+task examplesJavaIntegrationTest(type: Test) {
+ group = "Verification"
+ def dataflowProject = project.findProperty('dataflowProject') ?: 'apache-beam-testing'
+ def dataflowTempRoot = project.findProperty('dataflowTempRoot') ?: 'gs://temp-storage-for-end-to-end-tests'
+ systemProperty "beamTestPipelineOptions", JsonOutput.toJson([
+ "--runner=TestDataflowRunner",
+ "--project=${dataflowProject}",
+ "--tempRoot=${dataflowTempRoot}",
+ ])
+
+ // The examples/java preCommit task already covers running WordCountIT/WindowedWordCountIT so
+ // this postCommit integration test excludes them.
+ include '**/*IT.class'
+ exclude '**/WordCountIT.class'
+ exclude '**/WindowedWordCountIT.class'
+ maxParallelForks 4
+ classpath = configurations.googleCloudPlatformIntegrationTest
+ testClassesDirs = files(project(":beam-examples-java").sourceSets.test.output.classesDirs)
+ useJUnit { }
+}
+
+task postCommit {
+ group = "Verfication"
+ description = "Various integration tests using the Dataflow runner."
+ dependsOn googleCloudPlatformIntegrationTest
+ dependsOn examplesJavaIntegrationTest
+}
diff --git a/sdks/java/io/google-cloud-platform/build.gradle b/sdks/java/io/google-cloud-platform/build.gradle
index a292ebf0883..f9df0965f68 100644
--- a/sdks/java/io/google-cloud-platform/build.gradle
+++ b/sdks/java/io/google-cloud-platform/build.gradle
@@ -69,11 +69,33 @@ dependencies {
shadow library.java.proto_google_common_protos
shadow library.java.grpc_all
shadow library.java.netty_tcnative_boringssl_static
- testCompile project(path: ":beam-sdks-java-core", configuration: "shadowTest")
- testCompile project(":beam-sdks-java-extensions-google-cloud-platform-core").sourceSets.test.output
- testCompile project(path: ":beam-runners-direct-java", configuration: "shadow")
- testCompile library.java.hamcrest_core
- testCompile library.java.mockito_core
- testCompile library.java.junit
- testCompile library.java.slf4j_jdk14
+ shadowTest project(path: ":beam-sdks-java-core", configuration: "shadowTest")
+ shadowTest project(":beam-sdks-java-extensions-google-cloud-platform-core").sourceSets.test.output
+ shadowTest project(path: ":beam-runners-direct-java", configuration: "shadow")
+ shadowTest library.java.hamcrest_core
+ shadowTest library.java.mockito_core
+ shadowTest library.java.junit
+ shadowTest library.java.slf4j_jdk14
+}
+
+// Create a shaded test jar.
+task shadowTestJar(type: com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar) {
+ classifier = "shaded-tests"
+ from sourceSets.test.output
+ configurations = [project.configurations.testRuntime]
+ dependencies {
+ exclude(".*")
+ include(dependency(library.java.guava))
+ }
+ relocate("com.google.common", getJavaRelocatedPath("com.google.common")) {
+ // com.google.common is too generic, need to exclude guava-testlib
+ exclude "com.google.common.collect.testing.**"
+ exclude "com.google.common.escape.testing.**"
+ exclude "com.google.common.testing.**"
+ exclude "com.google.common.util.concurrent.testing.**"
+ }
+}
+
+artifacts {
+ shadowTest shadowTestJar
}
--
GitLab