Fix Too long Http connection pool timeout

Signed-off-by: Paolo Di Tommaso <[email protected]>
nextflow-io · Sep 2, 2023 · fa961e7 · fa961e7
1 parent d19cb0b
commit fa961e7
Show file tree

Hide file tree

Showing 2 changed files with 43 additions and 10 deletions.
diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/Launcher.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/Launcher.groovy
@@ -578,15 +578,28 @@ class Launcher {
      */
     private void setupEnvironment() {
 
-        setProxy('HTTP',System.getenv())
-        setProxy('HTTPS',System.getenv())
-        setProxy('FTP',System.getenv())
+        final env = System.getenv()
+        setProxy('HTTP',env)
+        setProxy('HTTPS',env)
+        setProxy('FTP',env)
 
-        setProxy('http',System.getenv())
-        setProxy('https',System.getenv())
-        setProxy('ftp',System.getenv())
+        setProxy('http',env)
+        setProxy('https',env)
+        setProxy('ftp',env)
 
-        setNoProxy(System.getenv())
+        setNoProxy(env)
+
+        setHttpClientProperties(env)
+    }
+
+    static void setHttpClientProperties(Map<String,String> env) {
+        // Set the httpclient connection pool timeout to 10 seconds.
+        // This required because the default is 20 minutes, which cause the error
+        // "HTTP/1.1 header parser received no bytes" when in some circumstances
+        // https://github.com/nextflow-io/nextflow/issues/3983#issuecomment-1702305137
+        System.setProperty("jdk.httpclient.keepalive.timeout", env.getOrDefault("NXF_JDK_HTTPCLIENT_KEEPALIVE_TIMEOUT","10"))
+        if( env.get("NXF_JDK_HTTPCLIENT_CONNECTIONPOOLSIZE") )
+            System.setProperty("jdk.httpclient.connectionPoolSize", env.get("NXF_JDK_HTTPCLIENT_CONNECTIONPOOLSIZE"))
     }
 
     /**
@@ -650,7 +663,6 @@ class Launcher {
      * @param args The program options as specified by the user on the CLI
      */
     static void main(String... args)  {
-
         final status = new Launcher() .command(args) .run()
         if( status )
             System.exit(status)

diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/LauncherTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/LauncherTest.groovy
@@ -16,12 +16,12 @@
 
 package nextflow.cli
 
-import spock.lang.Specification
-
 import java.nio.file.Files
 
 import com.beust.jcommander.DynamicParameter
 import com.beust.jcommander.Parameter
+import spock.lang.Specification
+import spock.lang.Unroll
 import spock.util.environment.RestoreSystemProperties
 import test.OutputCapture
 /**
@@ -427,6 +427,27 @@ class LauncherTest extends Specification {
 
     }
 
+    @RestoreSystemProperties
+    @Unroll
+    def 'should set http client timeout' () {
+        when:
+        Launcher.setHttpClientProperties(ENV)
+        then:
+        System.getProperty('jdk.httpclient.keepalive.timeout') == TIMEOUT
+        and:
+        System.getProperty('jdk.httpclient.connectionPoolSize') == POOLSIZE
+
+        where:
+        ENV                                             | TIMEOUT   | POOLSIZE
+        [:]                                             | '10'      | null
+        and:
+        [NXF_JDK_HTTPCLIENT_KEEPALIVE_TIMEOUT: '1']     | '1'       | null
+        [NXF_JDK_HTTPCLIENT_KEEPALIVE_TIMEOUT: '100']   | '100'     | null
+        and:
+        [NXF_JDK_HTTPCLIENT_CONNECTIONPOOLSIZE: '0']    | '10'      | '0'
+        [NXF_JDK_HTTPCLIENT_CONNECTIONPOOLSIZE: '99']   | '10'      | '99'
+    }
+
     def 'should make cli' () {
         given:
         def launcher = new Launcher()