Fix PTY by being more permissive of fcntl flags. Fixes #5004.

headius · headius · commit cce87769f03f · 2018-01-24T11:07:07.000-06:00
Note that this logic is still wrong when attempting to clear flags
since it only looks for them to be set. We should mimic CRuby's
fcntl logic when we have access to a real file descriptor, and
use the current logic (with fixes for clearing flags) as a
fallback only.
diff --git a/core/src/main/java/org/jruby/RubyIO.java b/core/src/main/java/org/jruby/RubyIO.java
@@ -2439,12 +2439,12 @@ public void setBlocking(boolean blocking) {
 
     @JRubyMethod(name = "fcntl")
     public IRubyObject fcntl(ThreadContext context, IRubyObject cmd) {
-        return ctl(context.runtime, cmd, null);
+        return ctl(context, cmd, null);
     }
 
     @JRubyMethod(name = "fcntl")
     public IRubyObject fcntl(ThreadContext context, IRubyObject cmd, IRubyObject arg) {
-        return ctl(context.runtime, cmd, arg);
+        return ctl(context, cmd, arg);
     }
 
     @JRubyMethod(name = "ioctl", required = 1, optional = 1)
@@ -2458,10 +2458,11 @@ public IRubyObject ioctl(ThreadContext context, IRubyObject[] args) {
             arg = context.runtime.getNil();
         }
 
-        return ctl(context.runtime, cmd, arg);
+        return ctl(context, cmd, arg);
     }
 
-    private IRubyObject ctl(Ruby runtime, IRubyObject cmd, IRubyObject arg) {
+    private IRubyObject ctl(ThreadContext context, IRubyObject cmd, IRubyObject arg) {
+        Ruby runtime = context.runtime;
         long realCmd = cmd.convertToInteger().getLongValue();
         long nArg = 0;
 
@@ -2494,25 +2495,31 @@ private IRubyObject ctl(Ruby runtime, IRubyObject cmd, IRubyObject arg) {
         //   for mode changes which should persist across fork() boundaries.  Since JVM has no fork
         //   this is not a problem for us.
         if (realCmd == FcntlLibrary.FD_CLOEXEC) {
-            close_on_exec_set(runtime.getCurrentContext(), runtime.getTrue());
+            close_on_exec_set(context, runtime.getTrue());
         } else if (realCmd == Fcntl.F_SETFD.intValue()) {
             if (arg != null && (nArg & FcntlLibrary.FD_CLOEXEC) == FcntlLibrary.FD_CLOEXEC) {
-                close_on_exec_set(runtime.getCurrentContext(), arg);
+                close_on_exec_set(context, arg);
             } else {
                 throw runtime.newNotImplementedError("F_SETFD only supports FD_CLOEXEC");
             }
         } else if (realCmd == Fcntl.F_GETFD.intValue()) {
-            return runtime.newFixnum(close_on_exec_p(runtime.getCurrentContext()).isTrue() ? FD_CLOEXEC : 0);
+            return runtime.newFixnum(close_on_exec_p(context).isTrue() ? FD_CLOEXEC : 0);
         } else if (realCmd == Fcntl.F_SETFL.intValue()) {
             if ((nArg & OpenFlags.O_NONBLOCK.intValue()) != 0) {
-                boolean block = (nArg & ModeFlags.NONBLOCK) != ModeFlags.NONBLOCK;
+                fptr.setBlocking(runtime, true);
+            } else {
+                fptr.setBlocking(runtime, false);
+            }
 
-                fptr.setBlocking(runtime, block);
+            if ((nArg & OpenFlags.O_CLOEXEC.intValue()) != 0) {
+                close_on_exec_set(context, context.tru);
             } else {
-                throw runtime.newNotImplementedError("F_SETFL only supports O_NONBLOCK");
+                close_on_exec_set(context, context.fals);
             }
         } else if (realCmd == Fcntl.F_GETFL.intValue()) {
-            return fptr.isBlocking() ? RubyFixnum.zero(runtime) : RubyFixnum.newFixnum(runtime, ModeFlags.NONBLOCK);
+            return runtime.newFixnum(
+                    (fptr.isBlocking() ? 0 : OpenFlags.O_NONBLOCK.intValue()) |
+                            (close_on_exec_p(context).isTrue() ? FD_CLOEXEC : 0));
         } else {
             throw runtime.newNotImplementedError("JRuby only supports F_SETFL and F_GETFL with NONBLOCK for fcntl/ioctl");
         }
