Backport "SIP 46 - read classpath from file, remove lib directory in …

…distribution" to 3.5.0 (#20908)

Backports #20631 to 3.5.0-RC3

dist/bin-native-overrides/cli-common-platform.bat

@@ -12,7 +12,11 @@ FOR /F "usebackq delims=" %%G IN ("%_PROG_HOME%\EXTRA_PROPERTIES") DO (
   )
 )
 
+@REM we didn't find it, so we should fail
+echo "ERROR: cli_version not found in EXTRA_PROPERTIES file"
+exit /b 1
+
 :foundCliVersion
 endlocal & set "SCALA_CLI_VERSION=%_SCALA_CLI_VERSION%"
 
-set SCALA_CLI_CMD_WIN="%_PROG_HOME%\bin\scala-cli.exe" "--cli-version" "%SCALA_CLI_VERSION%"
+set SCALA_CLI_CMD_WIN="%_PROG_HOME%\bin\scala-cli.exe" "--cli-version" "%SCALA_CLI_VERSION%"

dist/bin/common

@@ -6,67 +6,34 @@ source "$PROG_HOME/bin/common-shared"
 # * The code below is for Dotty
 # *-------------------------------------------------*/
 
-find_lib () {
-  for lib in "$PROG_HOME"/lib/$1 ; do
-    if [[ -f "$lib" ]]; then
-      if [ -n "$CYGPATHCMD" ]; then
-        "$CYGPATHCMD" -am "$lib"
-      elif [[ $mingw ||  $msys ]]; then
-        echo "$lib" | sed 's|/|\\\\|g'
-      else
-        echo "$lib"
+load_classpath () {
+  command="$1"
+  psep_pattern="$2"
+  __CLASS_PATH=""
+  while IFS= read -r line || [ -n "$line" ]; do
+    # jna-5 only appropriate for some combinations
+    if ! [[ ( -n ${conemu-} || -n ${msys-}) && "$line" == "*jna-5*" ]]; then
+      if [ -n "$__CLASS_PATH" ]; then
+        __CLASS_PATH+="$psep_pattern"
       fi
-      return
+      __CLASS_PATH+="$PROG_HOME/maven2/$line"
     fi
-  done
+  done < "$PROG_HOME/etc/$command.classpath"
+  echo "$__CLASS_PATH"
 }
 
-DOTTY_COMP=$(find_lib "*scala3-compiler*")
-DOTTY_INTF=$(find_lib "*scala3-interfaces*")
-DOTTY_LIB=$(find_lib "*scala3-library*")
-DOTTY_STAGING=$(find_lib "*scala3-staging*")
-DOTTY_TASTY_INSPECTOR=$(find_lib "*scala3-tasty-inspector*")
-TASTY_CORE=$(find_lib "*tasty-core*")
-SCALA_ASM=$(find_lib "*scala-asm*")
-SCALA_LIB=$(find_lib "*scala-library*")
-SBT_INTF=$(find_lib "*compiler-interface*")
-JLINE_READER=$(find_lib "*jline-reader-3*")
-JLINE_TERMINAL=$(find_lib "*jline-terminal-3*")
-JLINE_TERMINAL_JNA=$(find_lib "*jline-terminal-jna-3*")
-
-# jna-5 only appropriate for some combinations
-[[ ${conemu-} && ${msys-} ]] || JNA=$(find_lib "*jna-5*")
-
 compilerJavaClasspathArgs () {
-  # echo "dotty-compiler: $DOTTY_COMP"
-  # echo "dotty-interface: $DOTTY_INTF"
-  # echo "dotty-library: $DOTTY_LIB"
-  # echo "tasty-core: $TASTY_CORE"
-  # echo "scala-asm: $SCALA_ASM"
-  # echo "scala-lib: $SCALA_LIB"
-  # echo "sbt-intface: $SBT_INTF"
+  toolchain="$(load_classpath "scala" "$PSEP")"
+  toolchain_extra="$(load_classpath "with_compiler" "$PSEP")"
 
-  toolchain=""
-  toolchain+="$SCALA_LIB$PSEP"
-  toolchain+="$DOTTY_LIB$PSEP"
-  toolchain+="$SCALA_ASM$PSEP"
-  toolchain+="$SBT_INTF$PSEP"
-  toolchain+="$DOTTY_INTF$PSEP"
-  toolchain+="$DOTTY_COMP$PSEP"
-  toolchain+="$TASTY_CORE$PSEP"
-  toolchain+="$DOTTY_STAGING$PSEP"
-  toolchain+="$DOTTY_TASTY_INSPECTOR$PSEP"
-
-  # jine
-  toolchain+="$JLINE_READER$PSEP"
-  toolchain+="$JLINE_TERMINAL$PSEP"
-  toolchain+="$JLINE_TERMINAL_JNA$PSEP"
-  [ -n "${JNA-}" ] && toolchain+="$JNA$PSEP"
+  if [ -n "$toolchain_extra" ]; then
+    toolchain+="$PSEP$toolchain_extra"
+  fi
 
   if [ -n "${jvm_cp_args-}" ]; then
     jvm_cp_args="$toolchain$jvm_cp_args"
   else
-    jvm_cp_args="$toolchain$PSEP"
+    jvm_cp_args="$toolchain"
   fi
 }
 

dist/bin/common.bat

@@ -38,20 +38,6 @@ if not defined _PROG_HOME (
    set _EXITCODE=1
    goto :eof
 )
-set "_LIB_DIR=%_PROG_HOME%\lib"
+set "_ETC_DIR=%_PROG_HOME%\etc"
 
 set _PSEP=;
-
-for /f "delims=" %%f in ('dir /a-d /b "%_LIB_DIR%\*scala3-compiler*"')        do set "_SCALA3_COMP=%_LIB_DIR%\%%f"
-for /f "delims=" %%f in ('dir /a-d /b "%_LIB_DIR%\*scala3-interfaces*"')      do set "_SCALA3_INTF=%_LIB_DIR%\%%f"
-for /f "delims=" %%f in ('dir /a-d /b "%_LIB_DIR%\*scala3-library*"')         do set "_SCALA3_LIB=%_LIB_DIR%\%%f"
-for /f "delims=" %%f in ('dir /a-d /b "%_LIB_DIR%\*scala3-staging*"')         do set "_SCALA3_STAGING=%_LIB_DIR%\%%f"
-for /f "delims=" %%f in ('dir /a-d /b "%_LIB_DIR%\*scala3-tasty-inspector*"') do set "_SCALA3_TASTY_INSPECTOR=%_LIB_DIR%\%%f"
-for /f "delims=" %%f in ('dir /a-d /b "%_LIB_DIR%\*tasty-core*"')             do set "_TASTY_CORE=%_LIB_DIR%\%%f"
-for /f "delims=" %%f in ('dir /a-d /b "%_LIB_DIR%\*scala-asm*"')              do set "_SCALA_ASM=%_LIB_DIR%\%%f"
-for /f "delims=" %%f in ('dir /a-d /b "%_LIB_DIR%\*scala-library*"')          do set "_SCALA_LIB=%_LIB_DIR%\%%f"
-for /f "delims=" %%f in ('dir /a-d /b "%_LIB_DIR%\*compiler-interface*"')     do set "_SBT_INTF=%_LIB_DIR%\%%f"
-for /f "delims=" %%f in ('dir /a-d /b "%_LIB_DIR%\*jline-reader-3*"')         do set "_JLINE_READER=%_LIB_DIR%\%%f"
-for /f "delims=" %%f in ('dir /a-d /b "%_LIB_DIR%\*jline-terminal-3*"')       do set "_JLINE_TERMINAL=%_LIB_DIR%\%%f"
-for /f "delims=" %%f in ('dir /a-d /b "%_LIB_DIR%\*jline-terminal-jna-3*"')   do set "_JLINE_TERMINAL_JNA=%_LIB_DIR%\%%f"
-for /f "delims=" %%f in ('dir /a-d /b "%_LIB_DIR%\*jna-5*"')                  do set "_JNA=%_LIB_DIR%\%%f"

dist/bin/scalac.bat

@@ -88,29 +88,17 @@ goto :eof
 
 @rem output parameter: _JVM_CP_ARGS
 :compilerJavaClasspathArgs
-@rem echo scala3-compiler: %_SCALA3_COMP%
-@rem echo scala3-interface: %_SCALA3_INTF%
-@rem echo scala3-library: %_SCALA3_LIB%
-@rem echo tasty-core: %_TASTY_CORE%
-@rem echo scala-asm: %_SCALA_ASM%
-@rem echo scala-lib: %_SCALA_LIB%
-@rem echo sbt-intface: %_SBT_INTF%
-
-set "__TOOLCHAIN=%_SCALA_LIB%%_PSEP%"
-set "__TOOLCHAIN=%__TOOLCHAIN%%_SCALA3_LIB%%_PSEP%"
-set "__TOOLCHAIN=%__TOOLCHAIN%%_SCALA_ASM%%_PSEP%"
-set "__TOOLCHAIN=%__TOOLCHAIN%%_SBT_INTF%%_PSEP%"
-set "__TOOLCHAIN=%__TOOLCHAIN%%_SCALA3_INTF%%_PSEP%"
-set "__TOOLCHAIN=%__TOOLCHAIN%%_SCALA3_COMP%%_PSEP%"
-set "__TOOLCHAIN=%__TOOLCHAIN%%_TASTY_CORE%%_PSEP%"
-set "__TOOLCHAIN=%__TOOLCHAIN%%_SCALA3_STAGING%%_PSEP%"
-set "__TOOLCHAIN=%__TOOLCHAIN%%_SCALA3_TASTY_INSPECTOR%%_PSEP%"
-
-@rem # jline
-set "__TOOLCHAIN=%__TOOLCHAIN%%_JLINE_READER%%_PSEP%"
-set "__TOOLCHAIN=%__TOOLCHAIN%%_JLINE_TERMINAL%%_PSEP%"
-set "__TOOLCHAIN=%__TOOLCHAIN%%_JLINE_TERMINAL_JNA%%_PSEP%"
-set "__TOOLCHAIN=%__TOOLCHAIN%%_JNA%%_PSEP%"
+
+set "CP_FILE=%_ETC_DIR%\scala.classpath"
+call :loadClasspathFromFile %CP_FILE%
+set "__TOOLCHAIN=%_CLASS_PATH_RESULT%"
+
+set "CP_FILE=%_ETC_DIR%\with_compiler.classpath"
+call :loadClasspathFromFile %CP_FILE%
+
+if defined _CLASS_PATH_RESULT (
+    set "__TOOLCHAIN=%__TOOLCHAIN%%_PSEP%%_CLASS_PATH_RESULT%"
+)
 
 if defined _SCALA_CPATH (
     set "_JVM_CP_ARGS=%__TOOLCHAIN%%_SCALA_CPATH%"
@@ -119,6 +107,24 @@ if defined _SCALA_CPATH (
 )
 goto :eof
 
+@REM concatentate every line in "%_ARG_FILE%" with _PSEP
+@REM arg 1 - file to read
+:loadClasspathFromFile
+set _ARG_FILE=%1
+set _CLASS_PATH_RESULT=
+if exist "%_ARG_FILE%" (
+    for /f "usebackq delims=" %%i in ("%_ARG_FILE%") do (
+        set "_LIB=%_PROG_HOME%\maven2\%%i"
+        set "_LIB=!_LIB:/=\!"
+        if not defined _CLASS_PATH_RESULT (
+            set "_CLASS_PATH_RESULT=!_LIB!"
+        ) else (
+            set "_CLASS_PATH_RESULT=!_CLASS_PATH_RESULT!%_PSEP%!_LIB!"
+        )
+    )
+)
+goto :eof
+
 @rem #########################################################################
 @rem ## Cleanups
 

dist/bin/scaladoc

@@ -53,62 +53,7 @@ addScrip() {
 }
 
 classpathArgs () {
-  CLASS_PATH=""
-  CLASS_PATH+="$(find_lib "*scaladoc*")$PSEP"
-  CLASS_PATH+="$(find_lib "*scala3-compiler*")$PSEP"
-  CLASS_PATH+="$(find_lib "*scala3-interfaces*")$PSEP"
-  CLASS_PATH+="$(find_lib "*scala3-library*")$PSEP"
-  CLASS_PATH+="$(find_lib "*tasty-core*")$PSEP"
-  CLASS_PATH+="$(find_lib "*scala3-tasty-inspector*")$PSEP"
-  CLASS_PATH+="$(find_lib "*flexmark-0*")$PSEP"
-  CLASS_PATH+="$(find_lib "*flexmark*")$PSEP"
-  CLASS_PATH+="$(find_lib "*flexmark-ext-anchorlink*")$PSEP"
-  CLASS_PATH+="$(find_lib "*flexmark-ext-autolink*")$PSEP"
-  CLASS_PATH+="$(find_lib "*flexmark-ext-emoji*")$PSEP"
-  CLASS_PATH+="$(find_lib "*flexmark-ext-gfm-strikethrough*")$PSEP"
-  CLASS_PATH+="$(find_lib "*flexmark-ext-gfm-tasklist*")$PSEP"
-  CLASS_PATH+="$(find_lib "*flexmark-ext-wikilink*")$PSEP"
-  CLASS_PATH+="$(find_lib "*flexmark-ext-yaml-front-matter*")$PSEP"
-  CLASS_PATH+="$(find_lib "*flexmark-ext-tables*")$PSEP"
-  CLASS_PATH+="$(find_lib "*flexmark-ext-ins*")$PSEP"
-  CLASS_PATH+="$(find_lib "*flexmark-ext-superscript*")$PSEP"
-  CLASS_PATH+="$(find_lib "*flexmark-util*")$PSEP"
-  CLASS_PATH+="$(find_lib "*flexmark-util-ast*")$PSEP"
-  CLASS_PATH+="$(find_lib "*flexmark-util-data*")$PSEP"
-  CLASS_PATH+="$(find_lib "*flexmark-util-dependency*")$PSEP"
-  CLASS_PATH+="$(find_lib "*flexmark-util-misc*")$PSEP"
-  CLASS_PATH+="$(find_lib "*flexmark-util-format*")$PSEP"
-  CLASS_PATH+="$(find_lib "*flexmark-util-sequence*")$PSEP"
-  CLASS_PATH+="$(find_lib "*flexmark-util-builder*")$PSEP"
-  CLASS_PATH+="$(find_lib "*flexmark-util-collection*")$PSEP"
-  CLASS_PATH+="$(find_lib "*flexmark-util-visitor*")$PSEP"
-  CLASS_PATH+="$(find_lib "*flexmark-util-options*")$PSEP"
-  CLASS_PATH+="$(find_lib "*flexmark-util-html*")$PSEP"
-  CLASS_PATH+="$(find_lib "*flexmark-formatter*")$PSEP"
-  CLASS_PATH+="$(find_lib "*flexmark-ast*")$PSEP"
-  CLASS_PATH+="$(find_lib "*liqp*")$PSEP"
-  CLASS_PATH+="$(find_lib "*jsoup*")$PSEP"
-  CLASS_PATH+="$(find_lib "*jackson-dataformat-yaml*")$PSEP"
-  CLASS_PATH+="$(find_lib "*jackson-datatype-jsr310*")$PSEP"
-  CLASS_PATH+="$(find_lib "*strftime4j*")$PSEP"
-  CLASS_PATH+="$(find_lib "*scala-asm*")$PSEP"
-  CLASS_PATH+="$(find_lib "*compiler-interface*")$PSEP"
-  CLASS_PATH+="$(find_lib "*jline-reader*")$PSEP"
-  CLASS_PATH+="$(find_lib "*jline-terminal-3*")$PSEP"
-  CLASS_PATH+="$(find_lib "*jline-terminal-jna*")$PSEP"
-  CLASS_PATH+="$(find_lib "*flexmark-formatter*")$PSEP"
-  CLASS_PATH+="$(find_lib "*autolink-0.6*")$PSEP"
-  CLASS_PATH+="$(find_lib "*flexmark-jira-converter*")$PSEP"
-  CLASS_PATH+="$(find_lib "*antlr4*")$PSEP"
-  CLASS_PATH+="$(find_lib "*jackson-annotations*")$PSEP"
-  CLASS_PATH+="$(find_lib "*jackson-core*")$PSEP"
-  CLASS_PATH+="$(find_lib "*jackson-databind*")$PSEP"
-  CLASS_PATH+="$(find_lib "*snakeyaml*")$PSEP"
-  CLASS_PATH+="$(find_lib "*scala-library*")$PSEP"
-  CLASS_PATH+="$(find_lib "*protobuf-java*")$PSEP"
-  CLASS_PATH+="$(find_lib "*util-interface*")$PSEP"
-  CLASS_PATH+="$(find_lib "*jna-5*")$PSEP"
-  CLASS_PATH+="$(find_lib "*antlr4-runtime*")$PSEP"
+  CLASS_PATH="$(load_classpath "scaladoc" "$PSEP")"
 
   jvm_cp_args="-classpath \"$CLASS_PATH\""
 }

dist/bin/scaladoc.bat

@@ -105,60 +105,24 @@ goto :eof
 
 @rem output parameter: _CLASS_PATH
 :classpathArgs
-set "_LIB_DIR=%_PROG_HOME%\lib"
-set _CLASS_PATH=
+set "_ETC_DIR=%_PROG_HOME%\etc"
 @rem keep list in sync with bash script `bin\scaladoc` !
-call :updateClasspath "scaladoc"
-call :updateClasspath "scala3-compiler"
-call :updateClasspath "scala3-interfaces"
-call :updateClasspath "scala3-library"
-call :updateClasspath "tasty-core"
-call :updateClasspath "scala3-tasty-inspector"
-call :updateClasspath "flexmark-0"
-call :updateClasspath "flexmark-html-parser"
-call :updateClasspath "flexmark-ext-anchorlink"
-call :updateClasspath "flexmark-ext-autolink"
-call :updateClasspath "flexmark-ext-emoji"
-call :updateClasspath "flexmark-ext-gfm-strikethrough"
-call :updateClasspath "flexmark-ext-gfm-tables"
-call :updateClasspath "flexmark-ext-gfm-tasklist"
-call :updateClasspath "flexmark-ext-wikilink"
-call :updateClasspath "flexmark-ext-yaml-front-matter"
-call :updateClasspath "liqp"
-call :updateClasspath "jsoup"
-call :updateClasspath "jackson-dataformat-yaml"
-call :updateClasspath "jackson-datatype-jsr310"
-call :updateClasspath "strftime4j"
-call :updateClasspath "scala-asm"
-call :updateClasspath "compiler-interface"
-call :updateClasspath "jline-reader"
-call :updateClasspath "jline-terminal-3"
-call :updateClasspath "jline-terminal-jna"
-call :updateClasspath "flexmark-util"
-call :updateClasspath "flexmark-formatter"
-call :updateClasspath "autolink-0.6"
-call :updateClasspath "flexmark-jira-converter"
-call :updateClasspath "antlr4"
-call :updateClasspath "jackson-annotations"
-call :updateClasspath "jackson-core"
-call :updateClasspath "jackson-databind"
-call :updateClasspath "snakeyaml"
-call :updateClasspath "scala-library"
-call :updateClasspath "protobuf-java"
-call :updateClasspath "util-interface"
-call :updateClasspath "jna-5"
-call :updateClasspath "flexmark-ext-tables"
-call :updateClasspath "flexmark-ext-ins"
-call :updateClasspath "flexmark-ext-superscript"
-call :updateClasspath "antlr4-runtime"
+call :loadClasspathFromFile
 goto :eof
 
-@rem input parameter: %1=pattern for library file
-@rem output parameter: _CLASS_PATH
-:updateClasspath
-set "__PATTERN=%~1"
-for /f "delims=" %%f in ('dir /a-d /b "%_LIB_DIR%\*%__PATTERN%*" 2^>NUL') do (
-    set "_CLASS_PATH=!_CLASS_PATH!%_LIB_DIR%\%%f%_PSEP%"
+@REM concatentate every line in "%_ETC_DIR%\scaladoc.classpath" with _PSEP
+:loadClasspathFromFile
+set _CLASS_PATH=
+if exist "%_ETC_DIR%\scaladoc.classpath" (
+    for /f "usebackq delims=" %%i in ("%_ETC_DIR%\scaladoc.classpath") do (
+        set "_LIB=%_PROG_HOME%\maven2\%%i"
+        set "_LIB=!_LIB:/=\!"
+        if not defined _CLASS_PATH (
+            set "_CLASS_PATH=!_LIB!"
+        ) else (
+            set "_CLASS_PATH=!_CLASS_PATH!%_PSEP%!_LIB!"
+        )
+    )
 )
 goto :eof
 

project/Build.scala

@@ -2127,7 +2127,14 @@ object Build {
     republishRepo := target.value / "republish",
     packResourceDir += (republishRepo.value / "bin" -> "bin"),
     packResourceDir += (republishRepo.value / "maven2" -> "maven2"),
-    Compile / pack := (Compile / pack).dependsOn(republish).value,
+    packResourceDir += (republishRepo.value / "etc" -> "etc"),
+    republishCommandLibs +=
+      ("scala" -> List("scala3-interfaces", "scala3-compiler", "scala3-library", "tasty-core")),
+    republishCommandLibs +=
+      ("with_compiler" -> List("scala3-staging", "scala3-tasty-inspector", "^!scala3-interfaces", "^!scala3-compiler", "^!scala3-library", "^!tasty-core")),
+    republishCommandLibs +=
+      ("scaladoc" -> List("scala3-interfaces", "scala3-compiler", "scala3-library", "tasty-core", "scala3-tasty-inspector", "scaladoc")),
+    Compile / pack := republishPack.value,
   )
 
   lazy val dist = project.asDist(Bootstrapped)
@@ -2167,7 +2174,7 @@ object Build {
       republishBinOverrides += (dist / baseDirectory).value / "bin-native-overrides",
       republishFetchCoursier := (dist / republishFetchCoursier).value,
       republishExtraProps += ("cli_version" -> scalaCliLauncherVersion),
-      mappings += (republishRepo.value / "etc" / "EXTRA_PROPERTIES" -> "EXTRA_PROPERTIES"),
+      mappings += (republishRepo.value / "EXTRA_PROPERTIES" -> "EXTRA_PROPERTIES"),
       republishLaunchers +=
         ("scala-cli.exe" -> s"zip+https://github.com/VirtusLab/scala-cli/releases/download/v$scalaCliLauncherVersionWindows/scala-cli-x86_64-pc-win32.zip!/scala-cli.exe")
     )