Update microsoft-spark JARs to include Scala version (#728)

dotnet · Oct 9, 2020 · 80c745b · 80c745b
1 parent 2a6479d
commit 80c745b
Show file tree

Hide file tree

Showing 52 changed files with 9 additions and 8 deletions.
diff --git a/src/csharp/Microsoft.Spark.E2ETest/SparkFixture.cs b/src/csharp/Microsoft.Spark.E2ETest/SparkFixture.cs
@@ -150,7 +150,8 @@ private void BuildSparkCmd(out string filename, out string args)
             string scalaDir = Path.Combine(curDir, "..", "..", "..", "..", "..", "src", "scala");
             string jarDir = Path.Combine(scalaDir, jarPrefix, "target");
             string assemblyVersion = Assembly.GetExecutingAssembly().GetName().Version.ToString(3);
-            string jar = Path.Combine(jarDir, $"{jarPrefix}-{assemblyVersion}.jar");
+            string scalaVersion = (SparkSettings.Version.Major == 3) ? "2.12" : "2.11";
+            string jar = Path.Combine(jarDir, $"{jarPrefix}_{scalaVersion}-{assemblyVersion}.jar");
 
             if (!File.Exists(jar))
             {
@@ -180,7 +181,7 @@ private void BuildSparkCmd(out string filename, out string args)
         private string GetJarPrefix()
         {
             Version sparkVersion = SparkSettings.Version;
-            return $"microsoft-spark-{sparkVersion.Major}.{sparkVersion.Minor}.x";
+            return $"microsoft-spark-{sparkVersion.Major}-{sparkVersion.Minor}";
         }
     }
 

diff --git a/src/scala/microsoft-spark-2.3.x/pom.xml → src/scala/microsoft-spark-2-3/pom.xml b/src/scala/microsoft-spark-2.3.x/pom.xml → src/scala/microsoft-spark-2-3/pom.xml
@@ -6,7 +6,7 @@
     <artifactId>microsoft-spark</artifactId>
     <version>${microsoft-spark.version}</version>
   </parent>
-  <artifactId>microsoft-spark-2.3.x</artifactId>
+  <artifactId>microsoft-spark-2-3_2.11</artifactId>
   <inceptionYear>2019</inceptionYear>
   <properties>
     <encoding>UTF-8</encoding>

diff --git a/...che/spark/api/dotnet/CallbackClient.scala → ...che/spark/api/dotnet/CallbackClient.scala b/...che/spark/api/dotnet/CallbackClient.scala → ...che/spark/api/dotnet/CallbackClient.scala
diff --git a/...spark/api/dotnet/CallbackConnection.scala → ...spark/api/dotnet/CallbackConnection.scala b/...spark/api/dotnet/CallbackConnection.scala → ...spark/api/dotnet/CallbackConnection.scala
diff --git a/...ache/spark/api/dotnet/DotnetBackend.scala → ...ache/spark/api/dotnet/DotnetBackend.scala b/...ache/spark/api/dotnet/DotnetBackend.scala → ...ache/spark/api/dotnet/DotnetBackend.scala
diff --git a/...ark/api/dotnet/DotnetBackendHandler.scala → ...ark/api/dotnet/DotnetBackendHandler.scala b/...ark/api/dotnet/DotnetBackendHandler.scala → ...ark/api/dotnet/DotnetBackendHandler.scala
diff --git a/...he/spark/api/dotnet/DotnetException.scala → ...he/spark/api/dotnet/DotnetException.scala b/...he/spark/api/dotnet/DotnetException.scala → ...he/spark/api/dotnet/DotnetException.scala
diff --git a/...g/apache/spark/api/dotnet/DotnetRDD.scala → ...g/apache/spark/api/dotnet/DotnetRDD.scala b/...g/apache/spark/api/dotnet/DotnetRDD.scala → ...g/apache/spark/api/dotnet/DotnetRDD.scala
diff --git a/...che/spark/api/dotnet/JvmBridgeUtils.scala → ...che/spark/api/dotnet/JvmBridgeUtils.scala b/...che/spark/api/dotnet/JvmBridgeUtils.scala → ...che/spark/api/dotnet/JvmBridgeUtils.scala
diff --git a/...a/org/apache/spark/api/dotnet/SerDe.scala → ...a/org/apache/spark/api/dotnet/SerDe.scala b/...a/org/apache/spark/api/dotnet/SerDe.scala → ...a/org/apache/spark/api/dotnet/SerDe.scala
diff --git a/.../apache/spark/api/dotnet/ThreadPool.scala → .../apache/spark/api/dotnet/ThreadPool.scala b/.../apache/spark/api/dotnet/ThreadPool.scala → .../apache/spark/api/dotnet/ThreadPool.scala
diff --git a/...he/spark/deploy/dotnet/DotnetRunner.scala → ...he/spark/deploy/dotnet/DotnetRunner.scala b/...he/spark/deploy/dotnet/DotnetRunner.scala → ...he/spark/deploy/dotnet/DotnetRunner.scala
diff --git a/...apache/spark/internal/config/Dotnet.scala → ...apache/spark/internal/config/Dotnet.scala b/...apache/spark/internal/config/Dotnet.scala → ...apache/spark/internal/config/Dotnet.scala
diff --git a/...pache/spark/sql/api/dotnet/SQLUtils.scala → ...pache/spark/sql/api/dotnet/SQLUtils.scala b/...pache/spark/sql/api/dotnet/SQLUtils.scala → ...pache/spark/sql/api/dotnet/SQLUtils.scala
diff --git a/...org/apache/spark/sql/test/TestUtils.scala → ...org/apache/spark/sql/test/TestUtils.scala b/...org/apache/spark/sql/test/TestUtils.scala → ...org/apache/spark/sql/test/TestUtils.scala
diff --git a/.../org/apache/spark/util/dotnet/Utils.scala → .../org/apache/spark/util/dotnet/Utils.scala b/.../org/apache/spark/util/dotnet/Utils.scala → .../org/apache/spark/util/dotnet/Utils.scala
diff --git a/...t/scala/com/microsoft/scala/AppTest.scala → ...t/scala/com/microsoft/scala/AppTest.scala b/...t/scala/com/microsoft/scala/AppTest.scala → ...t/scala/com/microsoft/scala/AppTest.scala
diff --git a/src/scala/microsoft-spark-2.4.x/pom.xml → src/scala/microsoft-spark-2-4/pom.xml b/src/scala/microsoft-spark-2.4.x/pom.xml → src/scala/microsoft-spark-2-4/pom.xml
@@ -6,7 +6,7 @@
     <artifactId>microsoft-spark</artifactId>
     <version>${microsoft-spark.version}</version>
   </parent>
-  <artifactId>microsoft-spark-2.4.x</artifactId>
+  <artifactId>microsoft-spark-2-4_2.11</artifactId>
   <inceptionYear>2019</inceptionYear>
   <properties>
     <encoding>UTF-8</encoding>

diff --git a/...che/spark/api/dotnet/CallbackClient.scala → ...che/spark/api/dotnet/CallbackClient.scala b/...che/spark/api/dotnet/CallbackClient.scala → ...che/spark/api/dotnet/CallbackClient.scala
diff --git a/...spark/api/dotnet/CallbackConnection.scala → ...spark/api/dotnet/CallbackConnection.scala b/...spark/api/dotnet/CallbackConnection.scala → ...spark/api/dotnet/CallbackConnection.scala
diff --git a/...ache/spark/api/dotnet/DotnetBackend.scala → ...ache/spark/api/dotnet/DotnetBackend.scala b/...ache/spark/api/dotnet/DotnetBackend.scala → ...ache/spark/api/dotnet/DotnetBackend.scala
diff --git a/...ark/api/dotnet/DotnetBackendHandler.scala → ...ark/api/dotnet/DotnetBackendHandler.scala b/...ark/api/dotnet/DotnetBackendHandler.scala → ...ark/api/dotnet/DotnetBackendHandler.scala
diff --git a/...he/spark/api/dotnet/DotnetException.scala → ...he/spark/api/dotnet/DotnetException.scala b/...he/spark/api/dotnet/DotnetException.scala → ...he/spark/api/dotnet/DotnetException.scala
diff --git a/...g/apache/spark/api/dotnet/DotnetRDD.scala → ...g/apache/spark/api/dotnet/DotnetRDD.scala b/...g/apache/spark/api/dotnet/DotnetRDD.scala → ...g/apache/spark/api/dotnet/DotnetRDD.scala
diff --git a/...che/spark/api/dotnet/JvmBridgeUtils.scala → ...che/spark/api/dotnet/JvmBridgeUtils.scala b/...che/spark/api/dotnet/JvmBridgeUtils.scala → ...che/spark/api/dotnet/JvmBridgeUtils.scala
diff --git a/...a/org/apache/spark/api/dotnet/SerDe.scala → ...a/org/apache/spark/api/dotnet/SerDe.scala b/...a/org/apache/spark/api/dotnet/SerDe.scala → ...a/org/apache/spark/api/dotnet/SerDe.scala
diff --git a/.../apache/spark/api/dotnet/ThreadPool.scala → .../apache/spark/api/dotnet/ThreadPool.scala b/.../apache/spark/api/dotnet/ThreadPool.scala → .../apache/spark/api/dotnet/ThreadPool.scala
diff --git a/...he/spark/deploy/dotnet/DotnetRunner.scala → ...he/spark/deploy/dotnet/DotnetRunner.scala b/...he/spark/deploy/dotnet/DotnetRunner.scala → ...he/spark/deploy/dotnet/DotnetRunner.scala
diff --git a/...apache/spark/internal/config/Dotnet.scala → ...apache/spark/internal/config/Dotnet.scala b/...apache/spark/internal/config/Dotnet.scala → ...apache/spark/internal/config/Dotnet.scala
diff --git a/...k/sql/api/dotnet/DotnetForeachBatch.scala → ...k/sql/api/dotnet/DotnetForeachBatch.scala b/...k/sql/api/dotnet/DotnetForeachBatch.scala → ...k/sql/api/dotnet/DotnetForeachBatch.scala
diff --git a/...pache/spark/sql/api/dotnet/SQLUtils.scala → ...pache/spark/sql/api/dotnet/SQLUtils.scala b/...pache/spark/sql/api/dotnet/SQLUtils.scala → ...pache/spark/sql/api/dotnet/SQLUtils.scala
diff --git a/...org/apache/spark/sql/test/TestUtils.scala → ...org/apache/spark/sql/test/TestUtils.scala b/...org/apache/spark/sql/test/TestUtils.scala → ...org/apache/spark/sql/test/TestUtils.scala
diff --git a/.../org/apache/spark/util/dotnet/Utils.scala → .../org/apache/spark/util/dotnet/Utils.scala b/.../org/apache/spark/util/dotnet/Utils.scala → .../org/apache/spark/util/dotnet/Utils.scala
diff --git a/...t/scala/com/microsoft/scala/AppTest.scala → ...t/scala/com/microsoft/scala/AppTest.scala b/...t/scala/com/microsoft/scala/AppTest.scala → ...t/scala/com/microsoft/scala/AppTest.scala
diff --git a/src/scala/microsoft-spark-3.0.x/pom.xml → src/scala/microsoft-spark-3-0/pom.xml b/src/scala/microsoft-spark-3.0.x/pom.xml → src/scala/microsoft-spark-3-0/pom.xml
@@ -6,7 +6,7 @@
     <artifactId>microsoft-spark</artifactId>
     <version>${microsoft-spark.version}</version>
   </parent>
-  <artifactId>microsoft-spark-3.0.x</artifactId>
+  <artifactId>microsoft-spark-3-0_2.12</artifactId>
   <inceptionYear>2019</inceptionYear>
   <properties>
     <encoding>UTF-8</encoding>

diff --git a/...che/spark/api/dotnet/CallbackClient.scala → ...che/spark/api/dotnet/CallbackClient.scala b/...che/spark/api/dotnet/CallbackClient.scala → ...che/spark/api/dotnet/CallbackClient.scala
diff --git a/...spark/api/dotnet/CallbackConnection.scala → ...spark/api/dotnet/CallbackConnection.scala b/...spark/api/dotnet/CallbackConnection.scala → ...spark/api/dotnet/CallbackConnection.scala
diff --git a/...ache/spark/api/dotnet/DotnetBackend.scala → ...ache/spark/api/dotnet/DotnetBackend.scala b/...ache/spark/api/dotnet/DotnetBackend.scala → ...ache/spark/api/dotnet/DotnetBackend.scala
diff --git a/...ark/api/dotnet/DotnetBackendHandler.scala → ...ark/api/dotnet/DotnetBackendHandler.scala b/...ark/api/dotnet/DotnetBackendHandler.scala → ...ark/api/dotnet/DotnetBackendHandler.scala
diff --git a/...he/spark/api/dotnet/DotnetException.scala → ...he/spark/api/dotnet/DotnetException.scala b/...he/spark/api/dotnet/DotnetException.scala → ...he/spark/api/dotnet/DotnetException.scala
diff --git a/...g/apache/spark/api/dotnet/DotnetRDD.scala → ...g/apache/spark/api/dotnet/DotnetRDD.scala b/...g/apache/spark/api/dotnet/DotnetRDD.scala → ...g/apache/spark/api/dotnet/DotnetRDD.scala
diff --git a/...che/spark/api/dotnet/JvmBridgeUtils.scala → ...che/spark/api/dotnet/JvmBridgeUtils.scala b/...che/spark/api/dotnet/JvmBridgeUtils.scala → ...che/spark/api/dotnet/JvmBridgeUtils.scala
diff --git a/...a/org/apache/spark/api/dotnet/SerDe.scala → ...a/org/apache/spark/api/dotnet/SerDe.scala b/...a/org/apache/spark/api/dotnet/SerDe.scala → ...a/org/apache/spark/api/dotnet/SerDe.scala
diff --git a/.../apache/spark/api/dotnet/ThreadPool.scala → .../apache/spark/api/dotnet/ThreadPool.scala b/.../apache/spark/api/dotnet/ThreadPool.scala → .../apache/spark/api/dotnet/ThreadPool.scala
diff --git a/...he/spark/deploy/dotnet/DotnetRunner.scala → ...he/spark/deploy/dotnet/DotnetRunner.scala b/...he/spark/deploy/dotnet/DotnetRunner.scala → ...he/spark/deploy/dotnet/DotnetRunner.scala
diff --git a/...apache/spark/internal/config/Dotnet.scala → ...apache/spark/internal/config/Dotnet.scala b/...apache/spark/internal/config/Dotnet.scala → ...apache/spark/internal/config/Dotnet.scala
diff --git a/...k/sql/api/dotnet/DotnetForeachBatch.scala → ...k/sql/api/dotnet/DotnetForeachBatch.scala b/...k/sql/api/dotnet/DotnetForeachBatch.scala → ...k/sql/api/dotnet/DotnetForeachBatch.scala
diff --git a/...pache/spark/sql/api/dotnet/SQLUtils.scala → ...pache/spark/sql/api/dotnet/SQLUtils.scala b/...pache/spark/sql/api/dotnet/SQLUtils.scala → ...pache/spark/sql/api/dotnet/SQLUtils.scala
diff --git a/...org/apache/spark/sql/test/TestUtils.scala → ...org/apache/spark/sql/test/TestUtils.scala b/...org/apache/spark/sql/test/TestUtils.scala → ...org/apache/spark/sql/test/TestUtils.scala
diff --git a/.../org/apache/spark/util/dotnet/Utils.scala → .../org/apache/spark/util/dotnet/Utils.scala b/.../org/apache/spark/util/dotnet/Utils.scala → .../org/apache/spark/util/dotnet/Utils.scala
diff --git a/...t/scala/com/microsoft/scala/AppTest.scala → ...t/scala/com/microsoft/scala/AppTest.scala b/...t/scala/com/microsoft/scala/AppTest.scala → ...t/scala/com/microsoft/scala/AppTest.scala
diff --git a/src/scala/pom.xml b/src/scala/pom.xml
@@ -11,9 +11,9 @@
   </properties>
 
   <modules>
-    <module>microsoft-spark-2.3.x</module>
-    <module>microsoft-spark-2.4.x</module>
-    <module>microsoft-spark-3.0.x</module>
+    <module>microsoft-spark-2-3</module>
+    <module>microsoft-spark-2-4</module>
+    <module>microsoft-spark-3-0</module>
   </modules>
 
   <pluginRepositories>