Auto merge of #37677 - jsen-:master, r=alexcrichton

libstd: support creation of anonymous pipe on WinXP/2K3

`PIPE_REJECT_REMOTE_CLIENTS` flag is not supported on Windows < VISTA, and every invocation of `anon_pipe` including attempts to pipe `std::process::Child`'s stdio fails.
This PR should work around this issue by performing a runtime check of windows version and conditionally omitting this flag on "XP and friends".

Getting the version should be probably moved out of the function `anon_pipe` itself (the OS version does not often change during runtime :) ), but:
 - I didn't find any precedent for this and assuming there's not much overhead (I hope windows does not perform any heuristics to find out it's own version, just fills couple of fields in the struct).
 - the code path is not especially performance sensitive anyway.

Author: bors

src/libstd/sys/windows/c.rs

@@ -182,6 +182,7 @@ pub const ERROR_INVALID_HANDLE: DWORD = 6;
 pub const ERROR_NO_MORE_FILES: DWORD = 18;
 pub const ERROR_HANDLE_EOF: DWORD = 38;
 pub const ERROR_FILE_EXISTS: DWORD = 80;
+pub const ERROR_INVALID_PARAMETER: DWORD = 87;
 pub const ERROR_BROKEN_PIPE: DWORD = 109;
 pub const ERROR_CALL_NOT_IMPLEMENTED: DWORD = 120;
 pub const ERROR_INSUFFICIENT_BUFFER: DWORD = 122;

src/libstd/sys/windows/pipe.rs

@@ -43,6 +43,7 @@ pub fn anon_pipe() -> io::Result<(AnonPipe, AnonPipe)> {
         let reader;
         let mut name;
         let mut tries = 0;
+        let mut reject_remote_clients_flag = c::PIPE_REJECT_REMOTE_CLIENTS;
         loop {
             tries += 1;
             let key: u64 = rand::thread_rng().gen();
@@ -56,12 +57,12 @@ pub fn anon_pipe() -> io::Result<(AnonPipe, AnonPipe)> {
 
             let handle = c::CreateNamedPipeW(wide_name.as_ptr(),
                                              c::PIPE_ACCESS_INBOUND |
-                                              c::FILE_FLAG_FIRST_PIPE_INSTANCE |
-                                              c::FILE_FLAG_OVERLAPPED,
+                                             c::FILE_FLAG_FIRST_PIPE_INSTANCE |
+                                             c::FILE_FLAG_OVERLAPPED,
                                              c::PIPE_TYPE_BYTE |
-                                              c::PIPE_READMODE_BYTE |
-                                              c::PIPE_WAIT |
-                                              c::PIPE_REJECT_REMOTE_CLIENTS,
+                                             c::PIPE_READMODE_BYTE |
+                                             c::PIPE_WAIT |
+                                             reject_remote_clients_flag,
                                              1,
                                              4096,
                                              4096,
@@ -76,11 +77,27 @@ pub fn anon_pipe() -> io::Result<(AnonPipe, AnonPipe)> {
             //
             // Don't try again too much though as this could also perhaps be a
             // legit error.
+            // If ERROR_INVALID_PARAMETER is returned, this probably means we're
+            // running on pre-Vista version where PIPE_REJECT_REMOTE_CLIENTS is
+            // not supported, so we continue retrying without it. This implies
+            // reduced security on Windows versions older than Vista by allowing
+            // connections to this pipe from remote machines.
+            // Proper fix would increase the number of FFI imports and introduce
+            // significant amount of Windows XP specific code with no clean
+            // testing strategy
+            // for more info see https://github.com/rust-lang/rust/pull/37677
             if handle == c::INVALID_HANDLE_VALUE {
                 let err = io::Error::last_os_error();
-                if tries < 10 &&
-                   err.raw_os_error() == Some(c::ERROR_ACCESS_DENIED as i32) {
-                    continue
+                let raw_os_err = err.raw_os_error();
+                if tries < 10 {
+                    if raw_os_err == Some(c::ERROR_ACCESS_DENIED as i32) {
+                        continue
+                    } else if reject_remote_clients_flag != 0 &&
+                        raw_os_err == Some(c::ERROR_INVALID_PARAMETER as i32) {
+                        reject_remote_clients_flag = 0;
+                        tries -= 1;
+                        continue
+                    }
                 }
                 return Err(err)
             }