Merge remote-tracking branch 'upstream/master' into gradle

.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Apex.groovy

@@ -19,15 +19,22 @@
 import common_job_properties
 
 // This job runs the suite of ValidatesRunner tests against the Apex runner.
-mavenJob('beam_PostCommit_Java_ValidatesRunner_Apex') {
+job('beam_PostCommit_Java_ValidatesRunner_Apex_Gradle') {
   description('Runs the ValidatesRunner suite on the Apex runner.')
+  previousNames('beam_PostCommit_Java_ValidatesRunner_Apex')
   previousNames('beam_PostCommit_Java_RunnableOnService_Apex')
 
   // Set common parameters.
   common_job_properties.setTopLevelMainJobProperties(delegate)
 
-  // Set maven parameters.
-  common_job_properties.setMavenConfig(delegate)
+  def gradle_switches = [
+    // Gradle log verbosity enough to diagnose basic build issues
+    "--info",
+    // Continue the build even if there is a failure to show as many potential failures as possible.
+    '--continue',
+    // Until we verify the build cache is working appropriately, force rerunning all tasks
+    '--rerun-tasks',
+  ]
 
   // Sets that this is a PostCommit job.
   common_job_properties.setPostCommit(delegate)
@@ -38,11 +45,14 @@ mavenJob('beam_PostCommit_Java_ValidatesRunner_Apex') {
     'Apache Apex Runner ValidatesRunner Tests',
     'Run Apex ValidatesRunner')
 
-  // Maven goals for this job.
-  goals('''clean verify --projects runners/apex \
-      --also-make \
-      --batch-mode \
-      --errors \
-      --activate-profiles validates-runner-tests \
-      --activate-profiles local-validates-runner-tests''')
+  // Gradle goals for this job.
+  steps {
+    gradle {
+      rootBuildScriptDir(common_job_properties.checkoutDir)
+      tasks(':runners:apex:validatesRunner')
+      for (String gradle_switch : gradle_switches) {
+        switches(gradle_switch)
+      }
+    }
+  }
 }

.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Dataflow.groovy

@@ -20,16 +20,22 @@ import common_job_properties
 
 // This job runs the suite of ValidatesRunner tests against the Dataflow
 // runner.
-mavenJob('beam_PostCommit_Java_ValidatesRunner_Dataflow') {
+job('beam_PostCommit_Java_ValidatesRunner_Dataflow_Gradle') {
   description('Runs the ValidatesRunner suite on the Dataflow runner.')
+  previousNames('beam_PostCommit_Java_ValidatesRunner_Dataflow')
   previousNames('beam_PostCommit_Java_RunnableOnService_Dataflow')
 
-
   // Set common parameters. Sets a long (3 hour) timeout due to timeouts in [BEAM-3775].
   common_job_properties.setTopLevelMainJobProperties(delegate, 'master', 180)
 
-  // Set maven parameters.
-  common_job_properties.setMavenConfig(delegate)
+  def gradle_switches = [
+    // Gradle log verbosity enough to diagnose basic build issues
+    "--info",
+    // Continue the build even if there is a failure to show as many potential failures as possible.
+    '--continue',
+    // Until we verify the build cache is working appropriately, force rerunning all tasks
+    '--rerun-tasks',
+  ]
 
   // Sets that this is a PostCommit job.
   common_job_properties.setPostCommit(delegate)
@@ -40,6 +46,14 @@ mavenJob('beam_PostCommit_Java_ValidatesRunner_Dataflow') {
     'Google Cloud Dataflow Runner ValidatesRunner Tests',
     'Run Dataflow ValidatesRunner')
 
-  // Maven goals for this job.
-  goals('-B -e clean verify -am -pl runners/google-cloud-dataflow-java -DforkCount=0 -DvalidatesRunnerPipelineOptions=\'[ "--runner=TestDataflowRunner", "--project=apache-beam-testing", "--tempRoot=gs://temp-storage-for-validates-runner-tests/" ]\'')
+  // Gradle goals for this job.
+  steps {
+    gradle {
+      rootBuildScriptDir(common_job_properties.checkoutDir)
+      tasks(':runners:google-cloud-dataflow-java:validatesRunner')
+      for (String gradle_switch : gradle_switches) {
+        switches(gradle_switch)
+      }
+    }
+  }
 }

.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Flink.groovy

@@ -19,7 +19,7 @@
 import common_job_properties
 
 // This job runs the suite of ValidatesRunner tests against the Flink runner.
-job('beam_PostCommit_Java_ValidatesRunner_Flink') {
+job('beam_PostCommit_Java_ValidatesRunner_Flink_Gradle') {
   description('Runs the ValidatesRunner suite on the Flink runner.')
 
   // Set common parameters.

.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy

@@ -20,18 +20,24 @@ import common_job_properties
 
 // This job runs the suite of ValidatesRunner tests against the Gearpump
 // runner.
-mavenJob('beam_PostCommit_Java_ValidatesRunner_Gearpump') {
+job('beam_PostCommit_Java_ValidatesRunner_Gearpump_Gradle') {
   description('Runs the ValidatesRunner suite on the Gearpump runner.')
-
+  previousNames('beam_PostCommit_Java_ValidatesRunner_Gearpump')
   previousNames('beam_PostCommit_Java_RunnableOnService_Gearpump')
 
   // Set common parameters.
   common_job_properties.setTopLevelMainJobProperties(
       delegate,
       'gearpump-runner')
 
-  // Set maven parameters.
-  common_job_properties.setMavenConfig(delegate)
+  def gradle_switches = [
+    // Gradle log verbosity enough to diagnose basic build issues
+    "--info",
+    // Continue the build even if there is a failure to show as many potential failures as possible.
+    '--continue',
+    // Until we verify the build cache is working appropriately, force rerunning all tasks
+    '--rerun-tasks',
+  ]
 
   // Sets that this is a PostCommit job.
   // 0 5 31 2 * will run on Feb 31 (i.e. never) according to job properties.
@@ -44,6 +50,14 @@ mavenJob('beam_PostCommit_Java_ValidatesRunner_Gearpump') {
     'Apache Gearpump Runner ValidatesRunner Tests',
     'Run Gearpump ValidatesRunner')
 
-  // Maven goals for this job.
-  goals('-B -e clean verify -am -pl runners/gearpump -DforkCount=0 -DvalidatesRunnerPipelineOptions=\'[ "--runner=TestGearpumpRunner"]\'')
+  // Gradle goals for this job.
+  steps {
+    gradle {
+      rootBuildScriptDir(common_job_properties.checkoutDir)
+      tasks(':runners:gearpump:validatesRunner')
+      for (String gradle_switch : gradle_switches) {
+        switches(gradle_switch)
+      }
+    }
+  }
 }

.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Spark.groovy

@@ -19,16 +19,22 @@
 import common_job_properties
 
 // This job runs the suite of ValidatesRunner tests against the Spark runner.
-mavenJob('beam_PostCommit_Java_ValidatesRunner_Spark') {
+job('beam_PostCommit_Java_ValidatesRunner_Spark_Gradle') {
   description('Runs the ValidatesRunner suite on the Spark runner.')
-
+  previousNames('beam_PostCommit_Java_ValidatesRunner_Spark')
   previousNames('beam_PostCommit_Java_RunnableOnService_Spark')
 
   // Set common parameters.
   common_job_properties.setTopLevelMainJobProperties(delegate, 'master', 120)
 
-  // Set maven parameters.
-  common_job_properties.setMavenConfig(delegate)
+  def gradle_switches = [
+    // Gradle log verbosity enough to diagnose basic build issues
+    "--info",
+    // Continue the build even if there is a failure to show as many potential failures as possible.
+    '--continue',
+    // Until we verify the build cache is working appropriately, force rerunning all tasks
+    '--rerun-tasks',
+  ]
 
   // Sets that this is a PostCommit job.
   common_job_properties.setPostCommit(delegate)
@@ -39,6 +45,14 @@ mavenJob('beam_PostCommit_Java_ValidatesRunner_Spark') {
     'Apache Spark Runner ValidatesRunner Tests',
     'Run Spark ValidatesRunner')
 
-  // Maven goals for this job.
-  goals('-B -e clean verify -am -pl runners/spark -Plocal-validates-runner-tests -Dspark.ui.enabled=false')
+  // Gradle goals for this job.
+  steps {
+    gradle {
+      rootBuildScriptDir(common_job_properties.checkoutDir)
+      tasks(':runners:spark:validatesRunner')
+      for (String gradle_switch : gradle_switches) {
+        switches(gradle_switch)
+      }
+    }
+  }
 }

.test-infra/jenkins/job_beam_PreCommit_Python_GradleBuild.groovy

@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// This is the Python precommit which runs a Gradle build, and the current set
+// of precommit tests.
+job('beam_PreCommit_Python_GradleBuild') {
+  description('Runs Python PreCommit tests for the current GitHub Pull Request.')
+
+  // Execute concurrent builds if necessary.
+  concurrentBuild()
+
+  // Set common parameters.
+  common_job_properties.setTopLevelMainJobProperties(
+    delegate,
+    'master',
+    90)
+
+  def gradle_switches = [
+    // Gradle log verbosity enough to diagnose basic build issues
+    "--info",
+    // Continue the build even if there is a failure to show as many potential failures as possible.
+    '--continue',
+    // Until we verify the build cache is working appropriately, force rerunning all tasks
+    '--rerun-tasks',
+  ]
+
+  def gradle_command_line = './gradlew ' + gradle_switches.join(' ') + ' :pythonPreCommit'
+  // Sets that this is a PreCommit job.
+  common_job_properties.setPreCommit(delegate, gradle_command_line, 'Run Python Gradle PreCommit')
+  steps {
+    gradle {
+      rootBuildScriptDir(common_job_properties.checkoutDir)
+      tasks(':pythonPreCommit')
+      for (String gradle_switch : gradle_switches) {
+        switches(gradle_switch)
+      }
+    }
+  }
+}

build.gradle

@@ -126,3 +126,9 @@ task goPreCommit() {
   dependsOn ":rat"
   dependsOn ":beam-sdks-go:test"
 }
+
+task pythonPreCommit() {
+  dependsOn ":rat"
+  dependsOn ":sdks:python:check"
+}
+

build_rules.gradle

@@ -152,6 +152,7 @@ def apex_core_version = "3.6.0"
 def apex_malhar_version = "3.4.0"
 def postgres_version = "9.4.1212.jre7"
 def jaxb_api_version = "2.2.12"
+def kafka_version = "1.0.0"
 
 // A map of maps containing common libraries used per language. To use:
 // dependencies {
@@ -235,7 +236,8 @@ ext.library = [
     jaxb_api: "javax.xml.bind:jaxb-api:$jaxb_api_version",
     joda_time: "joda-time:joda-time:2.4",
     junit: "junit:junit:4.12",
-    kafka_clients: "org.apache.kafka:kafka-clients:1.0.0",
+    kafka_2_11: "org.apache.kafka:kafka_2.11:$kafka_version",
+    kafka_clients: "org.apache.kafka:kafka-clients:$kafka_version",
     malhar_library: "org.apache.apex:malhar-library:$apex_malhar_version",
     mockito_core: "org.mockito:mockito-core:1.9.5",
     netty_handler: "io.netty:netty-handler:$netty_version",

examples/java/src/main/java/org/apache/beam/examples/complete/TfIdf.java

@@ -119,13 +119,13 @@ public static Set<URI> listInputDocuments(Options options)
     }
 
     Set<URI> uris = new HashSet<>();
-    if (absoluteUri.getScheme().equals("file")) {
+    if ("file".equals(absoluteUri.getScheme())) {
       File directory = new File(absoluteUri);
       for (String entry : Optional.fromNullable(directory.list()).or(new String[] {})) {
         File path = new File(directory, entry);
         uris.add(path.toURI());
       }
-    } else if (absoluteUri.getScheme().equals("gs")) {
+    } else if ("gs".equals(absoluteUri.getScheme())) {
       GcsUtil gcsUtil = options.as(GcsOptions.class).getGcsUtil();
       URI gcsUriGlob = new URI(
           absoluteUri.getScheme(),
@@ -167,7 +167,7 @@ public PCollection<KV<URI, String>> expand(PBegin input) {
       //  - gs: URIs on the service
       for (final URI uri : uris) {
         String uriString;
-        if (uri.getScheme().equals("file")) {
+        if ("file".equals(uri.getScheme())) {
           uriString = new File(uri).getPath();
         } else {
           uriString = uri.toString();
@@ -221,7 +221,7 @@ public void processElement(ProcessContext c) {
                   String line = c.element().getValue();
                   for (String word : line.split("\\W+")) {
                     // Log INFO messages when the word “love” is found.
-                    if (word.toLowerCase().equals("love")) {
+                    if ("love".equals(word.toLowerCase())) {
                       LOG.info("Found {}", word.toLowerCase());
                     }
 

examples/java/src/main/java/org/apache/beam/examples/cookbook/TriggerExample.java

@@ -394,7 +394,7 @@ static class ExtractFlowInfo extends DoFn<String, KV<String, Integer>> {
     @ProcessElement
     public void processElement(ProcessContext c) throws Exception {
       String[] laneInfo = c.element().split(",");
-      if (laneInfo[0].equals("timestamp")) {
+      if ("timestamp".equals(laneInfo[0])) {
         // Header row
         return;
       }