Cygwin: allow open_setup to fail

Convert fhandler_base::open_setup to a (virtual) method that returns a
bool result.  For the moment, it and its overrides always return true.

winsup/cygwin/fhandler.cc

@@ -789,9 +789,10 @@ fhandler_base::fd_reopen (int, mode_t)
   return NULL;
 }
 
-void
+bool
 fhandler_base::open_setup (int)
 {
+  return true;
 }
 
 /* states:

winsup/cygwin/fhandler.h

@@ -355,7 +355,7 @@ class fhandler_base
   int open_null (int flags);
   virtual int open (int, mode_t);
   virtual fhandler_base *fd_reopen (int, mode_t);
-  virtual void open_setup (int flags);
+  virtual bool open_setup (int flags);
   void set_unique_id (int64_t u) { unique_id = u; }
   void set_unique_id () { NtAllocateLocallyUniqueId ((PLUID) &unique_id); }
 
@@ -1206,7 +1206,7 @@ public:
   select_record *select_except (select_stuff *);
   char *get_proc_fd_name (char *buf);
   int open (int flags, mode_t mode = 0);
-  void open_setup (int flags);
+  bool open_setup (int flags);
   void fixup_after_fork (HANDLE);
   int dup (fhandler_base *child, int);
   void set_close_on_exec (bool val);
@@ -2132,7 +2132,7 @@ private:
   bool use_archetype () const {return true;}
 
   int open (int flags, mode_t mode);
-  void open_setup (int flags);
+  bool open_setup (int flags);
   int dup (fhandler_base *, int);
 
   void __reg3 read (void *ptr, size_t& len);
@@ -2300,7 +2300,7 @@ class fhandler_pty_slave: public fhandler_pty_common
   HANDLE& get_handle_nat () { return io_handle_nat; }
 
   int open (int flags, mode_t mode = 0);
-  void open_setup (int flags);
+  bool open_setup (int flags);
   ssize_t __stdcall write (const void *ptr, size_t len);
   void __reg3 read (void *ptr, size_t& len);
   int init (HANDLE, DWORD, mode_t);
@@ -2399,7 +2399,7 @@ public:
   void doecho (const void *str, DWORD len);
   int accept_input ();
   int open (int flags, mode_t mode = 0);
-  void open_setup (int flags);
+  bool open_setup (int flags);
   ssize_t __stdcall write (const void *ptr, size_t len);
   void __reg3 read (void *ptr, size_t& len);
   int close ();

winsup/cygwin/fhandler_console.cc

@@ -1366,13 +1366,13 @@ fhandler_console::open (int flags, mode_t)
   return 1;
 }
 
-void
+bool
 fhandler_console::open_setup (int flags)
 {
   set_flags ((flags & ~O_TEXT) | O_BINARY);
   if (myself->set_ctty (this, flags) && !myself->cygstarted)
     init_console_handler (true);
-  fhandler_base::open_setup (flags);
+  return fhandler_base::open_setup (flags);
 }
 
 int

winsup/cygwin/fhandler_pipe.cc

@@ -191,10 +191,11 @@ out:
   return 0;
 }
 
-void
+bool
 fhandler_pipe::open_setup (int flags)
 {
-  fhandler_base::open_setup (flags);
+  if (!fhandler_base::open_setup (flags))
+    goto err;
   if (get_dev () == FH_PIPER && !read_mtx)
     {
       SECURITY_ATTRIBUTES *sa = sec_none_cloexec (flags);
@@ -211,6 +212,10 @@ fhandler_pipe::open_setup (int flags)
     }
   if (get_dev () == FH_PIPEW && !query_hdl)
     set_pipe_non_blocking (is_nonblocking ());
+  return true;
+
+err:
+  return false;
 }
 
 off_t

winsup/cygwin/fhandler_tty.cc

@@ -964,13 +964,13 @@ err_no_msg:
   return 0;
 }
 
-void
+bool
 fhandler_pty_slave::open_setup (int flags)
 {
   set_flags ((flags & ~O_TEXT) | O_BINARY);
   myself->set_ctty (this, flags);
   report_tty_counts (this, "opened", "");
-  fhandler_base::open_setup (flags);
+  return fhandler_base::open_setup (flags);
 }
 
 void
@@ -1947,14 +1947,14 @@ fhandler_pty_master::open (int flags, mode_t)
   return 1;
 }
 
-void
+bool
 fhandler_pty_master::open_setup (int flags)
 {
   set_flags ((flags & ~O_TEXT) | O_BINARY);
   char buf[sizeof ("opened pty master for ptyNNNNNNNNNNN")];
   __small_sprintf (buf, "opened pty master for pty%d", get_minor ());
   report_tty_counts (this, buf, "");
-  fhandler_base::open_setup (flags);
+  return fhandler_base::open_setup (flags);
 }
 
 off_t