[BEAM-4014] Replace the Java maven based PostCommit with a Gradle based version.

https://builds.apache.org/view/A-D/view/Beam/job/beam_PreCommit_Java_MavenInstall/19300/testReport/api/json
shows the list of ITs that were executed. Note that its all the examples/java ITs and the sdks/java/io/google-cloud-platform ITs.

.test-infra/jenkins/job_beam_PostCommit_Java_MavenInstall.groovy → .test-infra/jenkins/job_beam_PostCommit_Java_GradleBuild.groovy

@@ -20,10 +20,10 @@ import common_job_properties
 
 // This job runs the Java postcommit tests, including the suite of integration
 // tests.
-mavenJob('beam_PostCommit_Java_MavenInstall') {
-  description('Runs postcommit tests on the Java SDK.')
-
+job('beam_PostCommit_Java_GradleBuild') {
+  description('Runs PostCommit tests on the Java SDK.')
   previousNames('beam_PostCommit_MavenVerify')
+  previousNames('beam_PostCommit_Java_MavenInstall')
 
   // Execute concurrent builds if necessary.
   concurrentBuild()
@@ -31,35 +31,39 @@ mavenJob('beam_PostCommit_Java_MavenInstall') {
   // Set common parameters.
   common_job_properties.setTopLevelMainJobProperties(delegate, 'master', 240)
 
-  // Set maven parameters.
-  common_job_properties.setMavenConfig(delegate)
+  def gradle_switches = [
+    // Gradle log verbosity enough to diagnose basic build issues
+    "--info",
+    // Continue the build even if there is a failure to show as many potential failures as possible.
+    '--continue',
+    // Until we verify the build cache is working appropriately, force rerunning all tasks
+    '--rerun-tasks',
+    // Specify maven home on Jenkins, needed by Maven archetype integration tests.
+    '-Pmaven_home=/home/jenkins/tools/maven/apache-maven-3.5.2'
+  ]
+
+  // Publish all test results to Jenkins
+  publishers {
+    archiveJunit('**/build/test-results/**/*.xml')
+  }
 
   // Sets that this is a PostCommit job.
   common_job_properties.setPostCommit(delegate)
 
   // Allows triggering this build against pull requests.
   common_job_properties.enablePhraseTriggeringFromPullRequest(
-          delegate,
-          'Java SDK Post Commit Tests',
-          'Run Java PostCommit')
+      delegate,
+      'Java SDK Post Commit Tests',
+      'Run Java PostCommit')
 
-  // Maven goals for this job.
-  goals([
-      'clean',
-      'install',
-      '--projects sdks/java/core,runners/direct-java,sdks/java/fn-execution',
-      ' --also-make',
-      '--also-make-dependents',
-      '--batch-mode',
-      '--errors',
-      '--fail-at-end',
-      '-P release,dataflow-runner',
-      '-DrepoToken=$COVERALLS_REPO_TOKEN',
-      '-D skipITs=false',
-      '''-D integrationTestPipelineOptions=\'[ \
-          "--project=apache-beam-testing", \
-          "--tempRoot=gs://temp-storage-for-end-to-end-tests", \
-          "--runner=TestDataflowRunner" \
-        ]\' '''
-  ].join(' '))
+  // Gradle goals for this job.
+  steps {
+    gradle {
+      rootBuildScriptDir(common_job_properties.checkoutDir)
+      tasks(':javaPostCommit')
+      for (String gradle_switch : gradle_switches) {
+        switches(gradle_switch)
+      }
+    }
+  }
 }

.test-infra/jenkins/job_beam_PreCommit_Java_GradleBuild.groovy

@@ -44,7 +44,7 @@ job('beam_PreCommit_Java_GradleBuild') {
     '--continue',
     // Until we verify the build cache is working appropriately, force rerunning all tasks
     '--rerun-tasks',
-    // Specify maven home on Jenkins
+    // Specify maven home on Jenkins, needed by Maven archetype integration tests.
     '-Pmaven_home=/home/jenkins/tools/maven/apache-maven-3.5.2'
   ]
 

build.gradle

@@ -109,7 +109,7 @@ rat {
 }
 check.dependsOn rat
 
-// Define root PreCommit tasks simplifying what is needed
+// Define root pre/post commit tasks simplifying what is needed
 // to be specified on the commandline when executing locally.
 // This indirection also makes Jenkins use the branch of the PR
 // for the test definitions.
@@ -122,6 +122,11 @@ task javaPreCommit() {
   dependsOn ":beam-examples-java:preCommit"
 }
 
+task javaPostCommit() {
+  dependsOn ":javaPreCommit"
+  dependsOn ":beam-runners-google-cloud-dataflow-java:postCommit"
+}
+
 task goPreCommit() {
   dependsOn ":rat"
   dependsOn ":beam-sdks-go:test"

runners/google-cloud-dataflow-java/build.gradle

@@ -51,6 +51,8 @@ test {
 
 configurations {
   validatesRunner
+  examplesJavaIntegrationTest
+  googleCloudPlatformIntegrationTest
 }
 
 dependencies {
@@ -89,6 +91,10 @@ dependencies {
   shadowTest library.java.jackson_dataformat_yaml
   validatesRunner project(path: ":beam-sdks-java-core", configuration: "shadowTest")
   validatesRunner project(path: project.path, configuration: "shadow")
+  examplesJavaIntegrationTest project(path: project.path, configuration: "shadow")
+  examplesJavaIntegrationTest project(path: ":beam-examples-java", configuration: "shadowTest")
+  googleCloudPlatformIntegrationTest project(path: project.path, configuration: "shadow")
+  googleCloudPlatformIntegrationTest project(path: ":beam-sdks-java-io-google-cloud-platform", configuration: "shadowTest")
 }
 
 test {
@@ -120,7 +126,6 @@ task validatesRunnerTest(type: Test) {
     excludeCategories 'org.apache.beam.sdk.testing.UsesUnboundedPCollections'
     excludeCategories 'org.apache.beam.sdk.testing.UsesTestStream'
     excludeCategories 'org.apache.beam.sdk.testing.UsesParDoLifecycle'
-
   }
 }
 
@@ -134,3 +139,48 @@ task validatesRunner {
 def gcpProject = project.findProperty('gcpProject') ?: 'apache-beam-testing'
 def gcsBucket = project.findProperty('gcsBucket') ?: 'temp-storage-for-release-validation-tests/quickstart'
 createJavaQuickstartValidationTask(name: 'Dataflow', gcpProject: gcpProject, gcsBucket: gcsBucket)
+
+task googleCloudPlatformIntegrationTest(type: Test) {
+  group = "Verification"
+  def dataflowProject = project.findProperty('dataflowProject') ?: 'apache-beam-testing'
+  def dataflowTempRoot = project.findProperty('dataflowTempRoot') ?: 'gs://temp-storage-for-end-to-end-tests'
+  systemProperty "beamTestPipelineOptions", JsonOutput.toJson([
+          "--runner=TestDataflowRunner",
+          "--project=${dataflowProject}",
+          "--tempRoot=${dataflowTempRoot}",
+  ])
+
+  include '**/*IT.class'
+  maxParallelForks 4
+  classpath = configurations.googleCloudPlatformIntegrationTest
+  testClassesDirs = files(project(":beam-sdks-java-io-google-cloud-platform").sourceSets.test.output.classesDirs)
+  useJUnit { }
+}
+
+task examplesJavaIntegrationTest(type: Test) {
+  group = "Verification"
+  def dataflowProject = project.findProperty('dataflowProject') ?: 'apache-beam-testing'
+  def dataflowTempRoot = project.findProperty('dataflowTempRoot') ?: 'gs://temp-storage-for-end-to-end-tests'
+  systemProperty "beamTestPipelineOptions", JsonOutput.toJson([
+          "--runner=TestDataflowRunner",
+          "--project=${dataflowProject}",
+          "--tempRoot=${dataflowTempRoot}",
+  ])
+
+  // The examples/java preCommit task already covers running WordCountIT/WindowedWordCountIT so
+  // this postCommit integration test excludes them.
+  include '**/*IT.class'
+  exclude '**/WordCountIT.class'
+  exclude '**/WindowedWordCountIT.class'
+  maxParallelForks 4
+  classpath = configurations.googleCloudPlatformIntegrationTest
+  testClassesDirs = files(project(":beam-examples-java").sourceSets.test.output.classesDirs)
+  useJUnit { }
+}
+
+task postCommit {
+  group = "Verfication"
+  description = "Various integration tests using the Dataflow runner."
+  dependsOn googleCloudPlatformIntegrationTest
+  dependsOn examplesJavaIntegrationTest
+}

sdks/java/io/google-cloud-platform/build.gradle

@@ -69,11 +69,33 @@ dependencies {
   shadow library.java.proto_google_common_protos
   shadow library.java.grpc_all
   shadow library.java.netty_tcnative_boringssl_static
-  testCompile project(path: ":beam-sdks-java-core", configuration: "shadowTest")
-  testCompile project(":beam-sdks-java-extensions-google-cloud-platform-core").sourceSets.test.output
-  testCompile project(path: ":beam-runners-direct-java", configuration: "shadow")
-  testCompile library.java.hamcrest_core
-  testCompile library.java.mockito_core
-  testCompile library.java.junit
-  testCompile library.java.slf4j_jdk14
+  shadowTest project(path: ":beam-sdks-java-core", configuration: "shadowTest")
+  shadowTest project(":beam-sdks-java-extensions-google-cloud-platform-core").sourceSets.test.output
+  shadowTest project(path: ":beam-runners-direct-java", configuration: "shadow")
+  shadowTest library.java.hamcrest_core
+  shadowTest library.java.mockito_core
+  shadowTest library.java.junit
+  shadowTest library.java.slf4j_jdk14
+}
+
+// Create a shaded test jar.
+task shadowTestJar(type: com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar) {
+  classifier = "shaded-tests"
+  from sourceSets.test.output
+  configurations = [project.configurations.testRuntime]
+  dependencies {
+    exclude(".*")
+    include(dependency(library.java.guava))
+  }
+  relocate("com.google.common", getJavaRelocatedPath("com.google.common")) {
+    // com.google.common is too generic, need to exclude guava-testlib
+    exclude "com.google.common.collect.testing.**"
+    exclude "com.google.common.escape.testing.**"
+    exclude "com.google.common.testing.**"
+    exclude "com.google.common.util.concurrent.testing.**"
+  }
+}
+
+artifacts {
+  shadowTest shadowTestJar
 }