Add multiplex worker support (#481)

Replaced BazelWorker with a generic Worker subsystem.

examples/android/app/BUILD.bazel

@@ -7,10 +7,10 @@ android_binary(
     custom_package = "examples.android.app",
     incremental_dexing = 0,
     manifest = "src/main/AndroidManifest.xml",
-    multidex = "native",
     manifest_values = {
         "lib_name": "lib",
     },
+    multidex = "native",
     deps = [
         "//libKtAndroid:my_kt",
     ],
@@ -22,10 +22,10 @@ android_binary(
     custom_package = "examples.android.app2",
     incremental_dexing = 0,
     manifest = "src/main/AndroidManifest.xml",
-    multidex = "native",
     manifest_values = {
-		"lib_name": "lib2",
-	},
+        "lib_name": "lib2",
+    },
+    multidex = "native",
     deps = [
         "//libAndroid:my_android",
     ],
@@ -37,10 +37,10 @@ android_binary(
     custom_package = "examples.android.app3",
     incremental_dexing = 1,
     manifest = "src/main/AndroidManifest.xml",
-    multidex = "native",
     manifest_values = {
-		"lib_name": "lib",
-	},
+        "lib_name": "lib",
+    },
+    multidex = "native",
     deps = [
         "//libKtAndroid:my_kt",
     ],
@@ -53,4 +53,4 @@ build_test(
         ":app2.apk",
         ":app3.apk",
     ],
-)
+)

examples/android/bzl/BUILD.bazel

@@ -2,7 +2,7 @@ load("@io_bazel_rules_kotlin//kotlin:kotlin.bzl", "define_kt_toolchain")
 load("@io_bazel_rules_kotlin//kotlin/internal:opts.bzl", "kt_javac_options", "kt_kotlinc_options")
 
 kt_kotlinc_options(
-    name = "default_kotlinc_options"
+    name = "default_kotlinc_options",
 )
 
 kt_javac_options(
@@ -12,10 +12,10 @@ kt_javac_options(
 define_kt_toolchain(
     name = "experimental_toolchain",
     api_version = "1.4",
-    experimental_use_abi_jars = True,
-    experimental_strict_kotlin_deps = "warn",
-    experimental_report_unused_deps = "warn",
     experimental_reduce_classpath_mode = "KOTLINBUILDER_REDUCED",
+    experimental_report_unused_deps = "warn",
+    experimental_strict_kotlin_deps = "warn",
+    experimental_use_abi_jars = True,
     javac_options = ":default_javac_options",
     kotlinc_options = ":default_kotlinc_options",
     language_version = "1.4",

examples/android/libAndroid/BUILD.bazel

@@ -6,8 +6,8 @@ android_library(
     srcs = glob(["src/main/java/**/*.java"]),
     custom_package = "examples.android.lib2",
     manifest = "src/main/AndroidManifest.xml",
-    visibility = ["//visibility:public"],
     resource_files = glob(["res/**"]),
+    visibility = ["//visibility:public"],
     deps = [
         ":util",
         "@maven//:androidx_appcompat_appcompat",

examples/android/libKtAndroid/BUILD.bazel

@@ -27,12 +27,12 @@ kt_android_library(
         ":serialization_plugin",
         ":autovalue",
     ],
+    resource_files = glob(["res/**"]),
     tags = ["trace"],
     visibility = ["//visibility:public"],
-    resource_files = glob(["res/**"]),
     deps = [
         "@maven//:androidx_appcompat_appcompat",
         "@maven//:com_google_auto_value_auto_value_annotations",
         "@maven//:org_jetbrains_kotlinx_kotlinx_serialization_runtime",
     ],
-)
+)

kotlin/internal/jvm/compile.bzl

@@ -335,6 +335,7 @@ def _run_merge_jdeps_action(ctx, rule_kind, toolchains, jdeps, outputs):
         executable = toolchains.kt.jdeps_merger.files_to_run.executable,
         execution_requirements = {
             "supports-workers": "1",
+            "supports-multiplex-workers": "1",
         },
         arguments = [args],
         progress_message = progress_message,
@@ -481,7 +482,10 @@ def _run_kt_builder_action(
         input_manifests = input_manifests,
         outputs = [f for f in outputs.values()],
         executable = toolchains.kt.kotlinbuilder.files_to_run.executable,
-        execution_requirements = {"supports-workers": "1"},
+        execution_requirements = {
+            "supports-workers": "1",
+            "supports-multiplex-workers": "1",
+        },
         arguments = [args],
         progress_message = progress_message,
         env = {
@@ -578,7 +582,7 @@ def kt_jvm_produce_jar_actions(ctx, rule_kind):
         annotation_processors = _create_annotation_processing(
             annotation_processors = annotation_processors,
             ap_class_jar = kt_java_output_jar,
-            ap_source_jar = kt_generated_java_srcjar
+            ap_source_jar = kt_generated_java_srcjar,
         )
 
     # If this rule has any resources declared setup a zipper action to turn them into a jar.
@@ -643,7 +647,6 @@ def kt_jvm_produce_jar_actions(ctx, rule_kind):
         ),
     )
 
-
 def _run_kt_java_builder_actions(
         ctx,
         rule_kind,
@@ -698,8 +701,8 @@ def _run_kt_java_builder_actions(
             JavaInfo(
                 compile_jar = kapt_generated_stub_jar,
                 output_jar = kapt_generated_stub_jar,
-                neverlink = True
-            )
+                neverlink = True,
+            ),
         )
 
     java_infos = []
@@ -819,14 +822,14 @@ def _run_kt_java_builder_actions(
         annotation_processing = _create_annotation_processing(
             annotation_processors = annotation_processors,
             ap_class_jar = [jars.class_jar for jars in java_info.outputs.jars][0],
-            ap_source_jar = ap_generated_src_jar
-       )
+            ap_source_jar = ap_generated_src_jar,
+        )
 
     return struct(
-       output_jars = output_jars,
-       generated_src_jars = generated_src_jars,
-       annotation_processing = annotation_processing,
-   )
+        output_jars = output_jars,
+        generated_src_jars = generated_src_jars,
+        annotation_processing = annotation_processing,
+    )
 
 def _create_annotation_processing(annotation_processors, ap_class_jar, ap_source_jar):
     """Creates the annotation_processing field for Kt to match what JavaInfo

kotlin/internal/repositories/setup.bzl

@@ -16,6 +16,7 @@ load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_
 load("@rules_jvm_external//:defs.bzl", "maven_install")
 load("//kotlin/internal/repositories:http_java_proto_file.bzl", "http_java_proto_file")
 load("@io_bazel_stardoc//:setup.bzl", "stardoc_repositories")
+load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace")
 
 def kt_configure():
     """Setup dependencies. Must be called AFTER kt_download_local_dev_dependencies() """
@@ -40,10 +41,10 @@ def kt_configure():
             "javax.inject:javax.inject:1",
             "org.pantsbuild:jarjar:1.7.2",
             "org.jetbrains.kotlinx:atomicfu-js:0.14.0",
-            "org.jetbrains.kotlinx:kotlinx-coroutines-core:1.3.9",
-            "org.jetbrains.kotlinx:kotlinx-coroutines-core-js:1.3.9",
-            "org.jetbrains.kotlinx:kotlinx-coroutines-test:1.3.9",
-            "org.jetbrains.kotlinx:kotlinx-coroutines-debug:1.3.9",
+            "org.jetbrains.kotlinx:kotlinx-coroutines-core:1.4.2",
+            "org.jetbrains.kotlinx:kotlinx-coroutines-core-js:1.4.2",
+            "org.jetbrains.kotlinx:kotlinx-coroutines-test:1.4.2",
+            "org.jetbrains.kotlinx:kotlinx-coroutines-debug:1.4.2",
             "org.jetbrains.kotlinx:kotlinx-serialization-runtime:1.0-M1-1.4.0-rc",
         ],
         repositories = [
@@ -69,3 +70,5 @@ def kt_configure():
     )
 
     stardoc_repositories()
+
+    bazel_skylib_workspace()

kotlin/internal/toolchains.bzl

@@ -104,7 +104,7 @@ _kt_toolchain = rule(
         ),
         "kotlinbuilder": attr.label(
             doc = "the kotlin builder executable",
-            default = Label("//src/main/kotlin:builder"),
+            default = Label("//src/main/kotlin:build"),
             executable = True,
             allow_files = True,
             cfg = "host",

scripts/win.bat

@@ -10,7 +10,7 @@ robocopy %REAL_SRC% %TARGET_DIR% /mir /xo /xd bazel-* /NFL /NDL
  bazel ^
     --output_base "c:\tmp\bazel" ^
     --output_user_root "c:\tmp\bazel_rules_kotlin" ^
-    build //src/main/kotlin:builder
+    build //src/main/kotlin:build
 
  bazel ^
     --output_base "c:\tmp\bazel" ^

src/main/kotlin/BUILD

@@ -16,19 +16,11 @@ load("//kotlin:kotlin.bzl", "kt_jvm_library")
 load("//third_party:jarjar.bzl", "jar_jar")
 load("//kotlin/internal/utils:packager.bzl", "release_archive")
 
-java_binary(
-    name = "builder_raw",
-    create_executable = False,
-    runtime_deps = ["//src/main/kotlin/io/bazel/kotlin/builder"],
+exports_files(
+    ["shade.jarjar"],
+    visibility = ["//src:__subpackages__"]
 )
 
-# The builder artifact. Shaded to ensure that libraries it uses are not leaked to
-# the code it's running against (e.g. dagger)
-jar_jar(
-    name = "builder_jar_jar",
-    input_jar = ":builder_raw_deploy.jar",
-    rules = "shade.jarjar",
-)
 
 jar_jar(
     name = "skip-code-gen",
@@ -44,36 +36,16 @@ jar_jar(
     visibility = ["//visibility:public"],
 )
 
-java_binary(
-    name = "builder",
-    data = [
-        "//src/main/kotlin:skip-code-gen",
-        "//src/main/kotlin:jdeps-gen",
-        "//src/main/kotlin/io/bazel/kotlin/compiler",
-        "@com_github_jetbrains_kotlin//:lib/kotlin-compiler.jar",
-    ],
-    jvm_flags = [
-        "-XX:+IgnoreUnrecognizedVMOptions",
-        "--add-opens=java.base/java.nio=ALL-UNNAMED",
-        "--add-opens=java.base/java.lang=ALL-UNNAMED",
-        "--add-opens=jdk.jdeps/com.sun.tools.jdeps=ALL-UNNAMED",
-    ],
-    main_class = "io.bazel.kotlin.builder.KotlinBuilderMain",
+alias(
+    name = "build",
+    actual = "//src/main/kotlin/io/bazel/kotlin/builder/cmd:build",
     visibility = ["//visibility:public"],
-    runtime_deps = [":builder_jar_jar"],
 )
 
-java_binary(
+alias(
     name = "jdeps_merger",
-    jvm_flags = [
-        "-XX:+IgnoreUnrecognizedVMOptions",
-        "--add-opens=java.base/java.nio=ALL-UNNAMED",
-        "--add-opens=java.base/java.lang=ALL-UNNAMED",
-        "--add-opens=jdk.jdeps/com.sun.tools.jdeps=ALL-UNNAMED",
-    ],
-    main_class = "io.bazel.kotlin.builder.JdepsMergerMain",
+    actual = "//src/main/kotlin/io/bazel/kotlin/builder/cmd:merge_jdeps",
     visibility = ["//visibility:public"],
-    runtime_deps = [":builder_jar_jar"],
 )
 
 release_archive(
@@ -84,8 +56,8 @@ release_archive(
     ],
     package_dir = "src/main/kotlin",  # explicitly set the package directory, as there are no parent release_archives.
     src_map = {
-        ":builder_deploy.jar": "kotlin_worker.jar",
-        ":jdeps_merger_deploy.jar": "jdeps_merger_worker.jar",
+        "//src/main/kotlin/io/bazel/kotlin/builder/cmd:build_deploy.jar": "kotlin_worker.jar",
+        "//src/main/kotlin/io/bazel/kotlin/builder/cmd:merge_jdeps_deploy.jar": "jdeps_merger_worker.jar",
         "BUILD.release.bazel": "BUILD.bazel",
     },
     deps = [

src/main/kotlin/BUILD.release.bazel

@@ -33,21 +33,21 @@ java_import(
 )
 
 java_binary(
-    name = "builder",
+    name = "build",
     data = [
-        ":skip-code-gen",
         ":jdeps-gen",
+        ":skip-code-gen",
         "//src/main/kotlin/io/bazel/kotlin/compiler",
         "@com_github_jetbrains_kotlin//:lib/kotlin-compiler.jar",
     ],
-    main_class = "io.bazel.kotlin.builder.KotlinBuilderMain",
+    main_class = "io.bazel.kotlin.builder.cmd.Build",
     visibility = ["//visibility:public"],
     runtime_deps = [":worker"],
 )
 
 java_binary(
     name = "jdeps_merger",
-    main_class = "io.bazel.kotlin.builder.JdepsMergerMain",
+    main_class = "io.bazel.kotlin.builder.cmd.MergeJdeps",
     visibility = ["//visibility:public"],
     runtime_deps = [":jdeps_merger_worker"],
 )

src/main/kotlin/bootstrap.bzl

@@ -11,7 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-load("@rules_java//java:defs.bzl", "java_import", "java_library")
+load("@rules_java//java:defs.bzl", "java_binary", "java_import", "java_library")
+load("//third_party:jarjar.bzl", "jar_jar")
 load("//kotlin:kotlin.bzl", _for_ide = "kt_jvm_library")
 
 _BOOTSTRAP_LIB_ARGS = ["-jvm-target", "1.8"]
@@ -58,13 +59,15 @@ case "$$(uname -s)" in
         ;;
 esac
 NAME=%s
-CP="$$(join_by $$SEP $(locations :%s))"
+CP="%s"
 ARGS="%s"
 
-$(JAVA) -Xmx256M -Xms32M -noverify \
-  -cp $(location @com_github_jetbrains_kotlin//:kotlin-preloader) org.jetbrains.kotlin.preloading.Preloader \
-  -cp $(location @com_github_jetbrains_kotlin//:kotlin-compiler) org.jetbrains.kotlin.cli.jvm.K2JVMCompiler \
-  -cp $${CP} -d $(@D)/$${NAME}_temp.jar $${ARGS} $(SRCS)
+CMD="$(JAVA) -Xmx256M -Xms32M -noverify \
+      -cp $(location @com_github_jetbrains_kotlin//:kotlin-preloader) org.jetbrains.kotlin.preloading.Preloader \
+      -cp $(location @com_github_jetbrains_kotlin//:kotlin-compiler) org.jetbrains.kotlin.cli.jvm.K2JVMCompiler \
+      $$CP -d $(@D)/$${NAME}_temp.jar $${ARGS} $(SRCS)"
+
+$$CMD
 
 case "$(location @bazel_tools//tools/jdk:singlejar)" in
     *.jar)
@@ -82,7 +85,7 @@ $$SJ \
     --output $(OUTS)
 
 rm $(@D)/$${NAME}_temp.jar
-""" % (name, dep_label, " ".join(_BOOTSTRAP_LIB_ARGS))
+""" % (name, "-cp $$(join_by $$SEP $(locations :%s)) " % dep_label if deps + neverlink_deps else "", " ".join(_BOOTSTRAP_LIB_ARGS))
     native.genrule(
         name = jar_label,
         tools = [
@@ -117,3 +120,41 @@ rm $(@D)/$${NAME}_temp.jar
         deps = [_resolve_dep_label(d) for d in deps] + neverlink_deps,
         visibility = ["//visibility:private"],
     )
+
+def kt_bootstrap_binary(
+        name,
+        main_class,
+        runtime_library,
+        shade_rules,
+        data = [],
+        visibility = ["//visibility:public"]):
+    raw = name + "_raw"
+    jar_jared = name + "_jarjar"
+
+    java_binary(
+        name = raw,
+        create_executable = False,
+        runtime_deps = [runtime_library],
+    )
+
+    # Shaded to ensure that libraries it uses are not leaked to
+    # the code it's running against (e.g. dagger)
+    jar_jar(
+        name = jar_jared,
+        input_jar = ":" + raw + "_deploy.jar",
+        rules = shade_rules,
+    )
+
+    java_binary(
+        name = name,
+        data = data,
+        jvm_flags = [
+            "-XX:+IgnoreUnrecognizedVMOptions",
+            "--add-opens=java.base/java.nio=ALL-UNNAMED",
+            "--add-opens=java.base/java.lang=ALL-UNNAMED",
+            "--add-opens=jdk.jdeps/com.sun.tools.jdeps=ALL-UNNAMED",
+        ],
+        main_class = main_class,
+        visibility = visibility,
+        runtime_deps = [":" + jar_jared],
+    )