Negative values are always considered not prime.

Fixes jruby/jruby#4193. Fixes #107.
jruby · Sep 30, 2016 · 2678bab · 2678bab
1 parent 4458259
commit 2678bab
Showing 1 changed file with 12 additions and 2 deletions.
diff --git a/src/main/java/org/jruby/ext/openssl/BN.java b/src/main/java/org/jruby/ext/openssl/BN.java
@@ -623,9 +623,14 @@ public RubyFixnum num_bits_set(final ThreadContext context) {
     public IRubyObject prime_p(IRubyObject[] args) {
         final Ruby runtime = getRuntime();
         int argc = Arity.checkArgumentCount(runtime, args, 0, 1);
+
+        // negative numbers are always considered non-prime
+        if (this.value.signum() < 0) return runtime.getFalse();
+
+        int certainty = argc == 0 ? DEFAULT_CERTAINTY : RubyNumeric.fix2int(args[0]);
+
         // BigInteger#isProbablePrime will actually limit checks to a maximum of 50,
         // depending on bit count.
-        int certainty = argc == 0 ? DEFAULT_CERTAINTY : RubyNumeric.fix2int(args[0]);
         return runtime.newBoolean(this.value.isProbablePrime(certainty));
     }
 
@@ -635,9 +640,14 @@ public IRubyObject prime_p(IRubyObject[] args) {
     public IRubyObject prime_fasttest_p(IRubyObject[] args) {
         final Ruby runtime = getRuntime();
         int argc = Arity.checkArgumentCount(runtime, args, 0, 2);
+
+        // negative numbers are always considered non-prime
+        if (this.value.signum() < 0) return runtime.getFalse();
+
+        int certainty = argc == 0 ? DEFAULT_CERTAINTY : RubyNumeric.fix2int(args[0]);
+
         // BigInteger#isProbablePrime will actually limit checks to a maximum of 50,
         // depending on bit count.
-        int certainty = argc == 0 ? DEFAULT_CERTAINTY : RubyNumeric.fix2int(args[0]);
         return runtime.newBoolean(this.value.isProbablePrime(certainty));
     }