expose libuv windows verbatim and hidden process spawning (fix #13776)

NEWS.md

@@ -64,6 +64,10 @@ Library improvements
   * New `foreach` function for calling a function on every element of a collection when
     the results are not needed.
 
+  * `Cmd(cmd; ...)` now accepts new Windows-specific options `windows_verbatim`
+    (to alter Windows command-line generation) and `windows_hide` (to
+    suppress creation of new console windows) ([#13780]).
+
 Deprecated or removed
 ---------------------
 

base/docs/helpdb.jl

@@ -1355,13 +1355,6 @@ Get a backtrace object for the current program point.
 """
 backtrace
 
-doc"""
-    ignorestatus(command)
-
-Mark a command object so that running it will not throw an error if the result code is non-zero.
-"""
-ignorestatus
-
 doc"""
     reducedim(f, A, dims[, initial])
 
@@ -6180,13 +6173,6 @@ Commit all currently buffered writes to the given stream.
 """
 flush
 
-doc"""
-    detach(command)
-
-Mark a command object so that it will be run in a new process group, allowing it to outlive the julia process, and not have Ctrl-C interrupts passed to it.
-"""
-detach
-
 doc"""
     precompile(f,args::Tuple{Vararg{Any}})
 

base/process.jl

@@ -2,22 +2,76 @@
 
 abstract AbstractCmd
 
+# libuv process option flags
+const UV_PROCESS_WINDOWS_VERBATIM_ARGUMENTS = UInt8(1 << 2)
+const UV_PROCESS_DETACHED = UInt8(1 << 3)
+const UV_PROCESS_WINDOWS_HIDE = UInt8(1 << 4)
+
 immutable Cmd <: AbstractCmd
     exec::Vector{ByteString}
     ignorestatus::Bool
-    detach::Bool
+    flags::UInt32 # libuv process flags
     env::Union{Array{ByteString},Void}
     dir::UTF8String
     Cmd(exec::Vector{ByteString}) =
-        new(exec, false, false, nothing, "")
-    Cmd(cmd::Cmd, ignorestatus, detach, env, dir) =
-        new(cmd.exec, ignorestatus, detach, env,
+        new(exec, false, 0x00, nothing, "")
+    Cmd(cmd::Cmd, ignorestatus, flags, env, dir) =
+        new(cmd.exec, ignorestatus, flags, env,
             dir === cmd.dir ? dir : cstr(dir))
-    Cmd(cmd::Cmd; ignorestatus=cmd.ignorestatus, detach=cmd.detach, env=cmd.env, dir=cmd.dir) =
-        new(cmd.exec, ignorestatus, detach, env,
+    function Cmd(cmd::Cmd; ignorestatus::Bool=cmd.ignorestatus, env=cmd.env, dir::AbstractString=cmd.dir,
+                 detach::Bool=Bool(cmd.flags & UV_PROCESS_DETACHED),
+                 windows_verbatim::Bool=Bool(cmd.flags & UV_PROCESS_WINDOWS_VERBATIM_ARGUMENTS),
+                 windows_hide::Bool=Bool(cmd.flags & UV_PROCESS_WINDOWS_HIDE))
+        flags = detach*UV_PROCESS_DETACHED |
+                windows_verbatim*UV_PROCESS_WINDOWS_VERBATIM_ARGUMENTS |
+                windows_hide*UV_PROCESS_WINDOWS_HIDE
+        new(cmd.exec, ignorestatus, flags, byteenv(env),
             dir === cmd.dir ? dir : cstr(dir))
+    end
 end
 
+doc"""
+    Cmd(cmd::Cmd; ignorestatus, detach, windows_verbatim, windows_hide,
+                  env, dir)
+
+Construct a new `Cmd` object, representing an external program and
+arguments, from `cmd`, while changing the settings of the optional
+keyword arguments:
+
+* `ignorestatus::Bool`: If `true` (defaults to `false`), then the `Cmd`
+  will not throw an error if the return code is nonzero.
+* `detach::Bool`: If `true` (defaults to `false`), then the `Cmd` will be
+  run in a new process group, allowing it to outlive the `julia` process
+  and not have Ctrl-C passed to it.
+* `windows_verbatim::Bool`: If `true` (defaults to `false`), then on Windows
+  the `Cmd` will send a command-line string to the process with no quoting
+  or escaping of arguments, even arguments containing spaces.  (On Windows,
+  arguments are sent to a program as a single "command-line" string, and
+  programs are responsible for parsing it into arguments.  By default,
+  empty arguments and arguments with spaces or tabs are quoted with double
+  quotes `"` in the command line, and `\` or `"` are preceded by backslashes.
+  `windows_verbatim=true` is useful for launching programs that parse their
+  command line in nonstandard ways.)  Has no effect on non-Windows systems.
+* `windows_hide::Bool`: If `true` (defaults to `false`), then on Windows no
+  new console window is displayed when the `Cmd` is executed.  This has
+  no effect if a console is already open or on non-Windows systems.
+* `env`: Set environment variables to use when running the `Cmd`.  `env`
+  is either a dictionary mapping strings to strings, an array
+  of strings of the form `"var=val"`, an array or tuple of `"var"=>val`
+  pairs, or `nothing`.  In order to modify (rather than replace)
+  the existing environment, create `env` by `copy(ENV)` and then
+  set `env["var"]=val` as desired.
+* `dir::AbstractString`: Specify a working directory for the command (instead
+  of the current directory).
+
+For any keywords that are not specified, the current settings from `cmd` are
+used.   Normally, to create a `Cmd` object in the first place, one uses
+backticks, e.g.
+
+    Cmd(`echo "Hello world"`, ignorestatus=true, detach=false)
+"""
+Cmd
+
 immutable OrCmds <: AbstractCmd
     a::AbstractCmd
     b::AbstractCmd
@@ -136,11 +190,21 @@ function show(io::IO, cr::CmdRedirect)
     print(io, ")")
 end
 
+"""
+    ignorestatus(command)
 
+Mark a command object so that running it will not throw an error if the result code is non-zero.
+"""
 ignorestatus(cmd::Cmd) = Cmd(cmd, ignorestatus=true)
 ignorestatus(cmd::Union{OrCmds,AndCmds}) =
     typeof(cmd)(ignorestatus(cmd.a), ignorestatus(cmd.b))
-detach(cmd::Cmd) = Cmd(cmd, detach=true)
+
+"""
+    detach(command)
+
+Mark a command object so that it will be run in a new process group, allowing it to outlive the julia process, and not have Ctrl-C interrupts passed to it.
+"""
+detach(cmd::Cmd) = Cmd(cmd; detach=true)
 
 # like bytestring(s), but throw an error if s contains NUL, since
 # libuv requires NUL-terminated strings
@@ -151,21 +215,19 @@ function cstr(s)
     return bytestring(s)
 end
 
-function setenv{S<:ByteString}(cmd::Cmd, env::Array{S}; dir="")
-    byteenv = ByteString[cstr(x) for x in env]
-    return Cmd(cmd; env = byteenv, dir = dir)
-end
-function setenv(cmd::Cmd, env::Associative; dir="")
-    byteenv = ByteString[cstr(string(k)*"="*string(v)) for (k,v) in env]
-    return Cmd(cmd; env = byteenv, dir = dir)
-end
-function setenv{T<:AbstractString}(cmd::Cmd, env::Pair{T}...; dir="")
-    byteenv = ByteString[cstr(k*"="*string(v)) for (k,v) in env]
-    return Cmd(cmd; env = byteenv, dir = dir)
-end
-function setenv(cmd::Cmd; dir="")
-    return Cmd(cmd; dir = dir)
-end
+# convert various env representations into an array of "key=val" strings
+byteenv{S<:AbstractString}(env::AbstractArray{S}) =
+    ByteString[cstr(x) for x in env]
+byteenv(env::Associative) =
+    ByteString[cstr(string(k)*"="*string(v)) for (k,v) in env]
+byteenv(env::Void) = nothing
+byteenv{T<:AbstractString}(env::Union{AbstractVector{Pair{T}}, Tuple{Vararg{Pair{T}}}}) =
+    ByteString[cstr(k*"="*string(v)) for (k,v) in env]
+
+setenv(cmd::Cmd, env; dir="") = Cmd(cmd; env=byteenv(env), dir=dir)
+setenv{T<:AbstractString}(cmd::Cmd, env::Pair{T}...; dir="") =
+    setenv(cmd, env; dir=dir)
+setenv(cmd::Cmd; dir="") = Cmd(cmd; dir=dir)
 
 (&)(left::AbstractCmd, right::AbstractCmd) = AndCmds(left, right)
 redir_out(src::AbstractCmd, dest::AbstractCmd) = OrCmds(src, dest)
@@ -255,7 +317,7 @@ function _jl_spawn(cmd, argv, loop::Ptr{Void}, pp::Process,
          Ptr{Void}, Int32, Ptr{Void}, Int32, Ptr{Void}, Int32, Ptr{Ptr{UInt8}}, Ptr{UInt8}, Ptr{Void}),
         cmd, argv, loop, proc, pp, uvtype(in),
         uvhandle(in), uvtype(out), uvhandle(out), uvtype(err), uvhandle(err),
-        pp.cmd.detach, pp.cmd.env === nothing ? C_NULL : pp.cmd.env, isempty(pp.cmd.dir) ? C_NULL : pp.cmd.dir,
+        pp.cmd.flags, pp.cmd.env === nothing ? C_NULL : pp.cmd.env, isempty(pp.cmd.dir) ? C_NULL : pp.cmd.dir,
         uv_jl_return_spawn::Ptr{Void})
     if error != 0
         ccall(:jl_forceclose_uv, Void, (Ptr{Void},), proc)

doc/stdlib/base.rst

@@ -850,6 +850,26 @@ System
 
    Mark a command object so that it will be run in a new process group, allowing it to outlive the julia process, and not have Ctrl-C interrupts passed to it.
 
+.. function:: Cmd(cmd::Cmd; ignorestatus, detach, windows_verbatim, windows_hide,
+                            env, dir)
+
+   .. Docstring generated from Julia source
+
+   Construct a new ``Cmd`` object, representing an external program and arguments, from ``cmd``\ , while changing the settings of the optional keyword arguments:
+
+   * ``ignorestatus::Bool``\ : If ``true`` (defaults to ``false``\ ), then the ``Cmd``   will not throw an error if the return code is nonzero.
+   * ``detach::Bool``\ : If ``true`` (defaults to ``false``\ ), then the ``Cmd`` will be   run in a new process group, allowing it to outlive the ``julia`` process   and not have Ctrl-C passed to it.
+   * ``windows_verbatim::Bool``\ : If ``true`` (defaults to ``false``\ ), then on Windows   the ``Cmd`` will send a command-line string to the process with no quoting   or escaping of arguments, even arguments containing spaces.  (On Windows,   arguments are sent to a program as a single "command-line" string, and   programs are responsible for parsing it into arguments.  By default,   empty arguments and arguments with spaces or tabs are quoted with double   quotes ``"`` in the command line, and ``\`` or ``"`` are preceded by backslashes.   ``windows_verbatim=true`` is useful for launching programs that parse their   command line in nonstandard ways.)  Has no effect on non-Windows systems.
+   * ``windows_hide::Bool``\ : If ``true`` (defaults to ``false``\ ), then on Windows no   new console window is displayed when the ``Cmd`` is executed.  This has   no effect if a console is already open or on non-Windows systems.
+   * ``env``\ : Set environment variables to use when running the ``Cmd``\ .  ``env``          is either a dictionary mapping strings to strings, an array          of strings of the form ``"var=val"``\ , an array or tuple of ``"var"=>val``          pairs, or ``nothing``\ .  In order to modify (rather than replace)          the existing environment, create ``env`` by ``copy(ENV)`` and then          set ``env["var"]=val`` as desired.
+   * ``dir::AbstractString``\ : Specify a working directory for the command (instead    of the current directory).
+
+   For any keywords that are not specified, the current settings from ``cmd`` are used.   Normally, to create a ``Cmd`` object in the first place, one uses backticks, e.g.
+
+   .. code-block:: julia
+
+       Cmd(`echo "Hello world"`, ignorestatus=true, detach=false)
+
 .. function:: setenv(command, env; dir=working_dir)
 
    .. Docstring generated from Julia source

doc/stdlib/io-network.rst

@@ -237,7 +237,7 @@ General I/O
 
    .. Docstring generated from Julia source
 
-   Write an arbitrary value to a stream in an opaque format, such that it can be read back by ``deserialize``\ . The read-back value will be as identical as possible to the original. In general, this process will not work if the reading and writing are done by different versions of Julia, or an instance of Julia with a different system image.
+   Write an arbitrary value to a stream in an opaque format, such that it can be read back by ``deserialize``\ . The read-back value will be as identical as possible to the original. In general, this process will not work if the reading and writing are done by different versions of Julia, or an instance of Julia with a different system image. ``Ptr`` values are serialized as all-zero bit patterns (``NULL``\ ).
 
 .. function:: deserialize(stream)
 

src/jl_uv.c

@@ -212,22 +212,20 @@ DLLEXPORT int jl_spawn(char *name, char **argv, uv_loop_t *loop,
                        uv_handle_type stdin_type, uv_pipe_t *stdin_pipe,
                        uv_handle_type stdout_type, uv_pipe_t *stdout_pipe,
                        uv_handle_type stderr_type, uv_pipe_t *stderr_pipe,
-                       int detach, char **env, char *cwd, uv_exit_cb cb)
+                       int flags, char **env, char *cwd, uv_exit_cb cb)
 {
     uv_process_options_t opts;
     uv_stdio_container_t stdio[3];
     int error;
     opts.file = name;
     opts.env = env;
 #ifdef _OS_WINDOWS_
-    opts.flags = 0;
+    opts.flags = flags;
 #else
-    opts.flags = UV_PROCESS_RESET_SIGPIPE;
+    opts.flags = flags | UV_PROCESS_RESET_SIGPIPE;
 #endif
     opts.cwd = cwd;
     opts.args = argv;
-    if (detach)
-        opts.flags |= UV_PROCESS_DETACHED;
     opts.stdio = stdio;
     opts.stdio_count = 3;
     stdio[0].type = stdin_type;

test/spawn.jl

@@ -334,3 +334,6 @@ end
 
 # issue #13616
 @test_throws ErrorException collect(eachline(`cat _doesnt_exist__111_`))
+
+# make sure windows_verbatim strips quotes
+@windows_only readall(`cmd.exe /c dir /b spawn.jl`) == readall(Cmd(`cmd.exe /c dir /b "\"spawn.jl\""`, windows_verbatim=true))