Merge pull request #3800 from jruby/truffle-cexts

[Truffle] Begin Sulong integration
jruby · Apr 13, 2016 · 6426629 · 6426629
2 parents 5fd9fe0 + a05f8c1
commit 6426629
Showing 14 changed files with 193 additions and 18 deletions.
diff --git a/.gitignore b/.gitignore
@@ -14,6 +14,8 @@
 *.zip
 *~
 *.tokens
+*.su
+*.ll
 
 .DS_Store
 .debug.properties

diff --git a/bin/jruby-cext-c b/bin/jruby-cext-c
@@ -0,0 +1,58 @@
+#!/usr/bin/env jruby
+
+# Copyright (c) 2016 Oracle and/or its affiliates. All rights reserved.
+# This code is released under a tri EPL/GPL/LGPL license. You can use it,
+# redistribute it and/or modify it under the terms of the:
+#
+# Eclipse Public License version 1.0
+# GNU General Public License version 2
+# GNU Lesser General Public License version 2.1
+
+# A compiler for JRuby+Truffle C extensions
+
+require 'yaml'
+
+MX_DIR = ENV['MX_DIR']
+
+unless MX_DIR
+  abort 'You need to set MX_DIR to a checkout of the mx repository'
+end
+
+SULONG_DIR = ENV['SULONG_DIR']
+
+unless SULONG_DIR
+  abort 'You need to set SULONG_DIR to the location of a built checkout of the Sulong repository'
+end
+
+def mx(command, *args)
+  command = "#{MX_DIR}/mx.py -p #{SULONG_DIR} --vm server #{command} #{args.join(' ')}"
+  puts "$ #{command}"
+  `#{command}`
+end
+
+CEXT_DIR = ARGV[0]
+
+unless CEXT_DIR
+  abort 'You need to pass the directory of a C extension so I can build it'
+end
+
+CONFIG_FILE = File.join(CEXT_DIR, '.jruby-cext-build.yml')
+
+unless File.exist?(CONFIG_FILE)
+  abort "There is no .jruby-cext-build.yml in this C extension at the moment - I don't know how to build it"
+end
+
+CONFIG = YAML.load_file(CONFIG_FILE)
+
+SRC = Dir[File.join(CEXT_DIR, CONFIG['src'])]
+OUT = File.join(CEXT_DIR, CONFIG['out'])
+
+LL = []
+
+SRC.each do |src|
+  ll = File.join(File.dirname(src), File.basename(src, '.*') + '.ll')
+  mx 'su-clang', '-Ilib/ruby/truffle/cext', '-S', '-emit-llvm', src, '-o', ll
+  LL.push ll
+end
+
+mx 'su-link', '-o', OUT, *LL
diff --git a/.../org/jruby/util/RubyFiletypeDetector.java → .../org/jruby/util/RubyFileTypeDetector.java b/.../org/jruby/util/RubyFiletypeDetector.java → .../org/jruby/util/RubyFileTypeDetector.java
@@ -9,7 +9,7 @@
 import java.util.Locale;
 import java.util.regex.Pattern;
 
-public final class RubyFiletypeDetector extends FileTypeDetector {
+public final class RubyFileTypeDetector extends FileTypeDetector {
 
     private static final String[] KNOWN_RUBY_FILES = new String[]{ "Gemfile", "Rakefile", "Mavenfile" };
     private static final String[] KNOWN_RUBY_SUFFIXES = new String[]{ ".rb", ".rake", ".gemspec" };
@@ -43,6 +43,8 @@ public String probeContentType(Path path) throws IOException {
             if (firstLine != null && SHEBANG_REGEXP.matcher(firstLine).matches()) {
                 return RUBY_MIME;
             }
+        } catch (IOException e) {
+            // Reading random files as UTF-8 could cause all sorts of errors
         }
 
         return null;

diff --git a/core/src/main/resources/META-INF/services/java.nio.file.spi.FileTypeDetector b/core/src/main/resources/META-INF/services/java.nio.file.spi.FileTypeDetector
@@ -1 +1 @@
-org.jruby.util.RubyFiletypeDetector
+org.jruby.util.RubyFileTypeDetector
diff --git a/test/truffle/cexts/minimum/.jruby-cext-build.yml b/test/truffle/cexts/minimum/.jruby-cext-build.yml
@@ -0,0 +1,2 @@
+src: ext/minimum/*.c
+out: lib/minimum/minimum.su
diff --git a/test/truffle/cexts/minimum/bin/minimum b/test/truffle/cexts/minimum/bin/minimum
@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+
+require 'minimum'
diff --git a/test/truffle/cexts/minimum/ext/minimum/minimum.c b/test/truffle/cexts/minimum/ext/minimum/minimum.c
@@ -0,0 +1,5 @@
+#include <stdio.h>
+
+void Init_minimum() {
+  printf("Hello!\n");
+}
diff --git a/test/truffle/cexts/minimum/lib/minimum.rb b/test/truffle/cexts/minimum/lib/minimum.rb
@@ -0,0 +1 @@
+require 'minimum/minimum'
diff --git a/test/truffle/cexts/minimum/lib/minimum/.keep b/test/truffle/cexts/minimum/lib/minimum/.keep
diff --git a/tool/jt.rb b/tool/jt.rb
@@ -61,6 +61,12 @@ def self.find_graal_js
     raise "couldn't find trufflejs.jar - download GraalVM as described in https://github.com/jruby/jruby/wiki/Downloading-GraalVM and find it in there"
   end
 
+  def self.find_sulong_dir
+    dir = ENV['SULONG_DIR']
+    return dir if dir
+    raise "couldn't find the Sulong repository - you need to check it out and build it"
+  end
+
   def self.jruby_eclipse?
     # tool/jruby_eclipse only works on release currently
     ENV["JRUBY_ECLIPSE"] == "true" && Utilities.git_branch == "master"
@@ -258,6 +264,7 @@ def help
     puts 'jt run [options] args...                       run JRuby with -X+T and args'
     puts '    --graal         use Graal (set GRAAL_BIN or it will try to automagically find it)'
     puts '    --js            add Graal.js to the classpath (set GRAAL_JS_JAR)'
+    puts '    --sulong        add Sulong to the classpath (set SULONG_JAR, implies --graal)'
     puts '    --asm           show assembly (implies --graal)'
     puts '    --server        run an instrumentation server on port 8080'
     puts '    --igv           make sure IGV is running and dump Graal graphs after partial escape (implies --graal)'
@@ -305,6 +312,7 @@ def help
     puts '  GRAAL_BIN_...git_branch_name...              GraalVM executable to use for a given branch'
     puts '           branch names are mangled - eg truffle-head becomes GRAAL_BIN_TRUFFLE_HEAD'
     puts '  GRAAL_JS_JAR                                 The location of trufflejs.jar'
+    puts '  SULONG_DIR                                   The location of a built checkout of the Sulong repository'
   end
 
   def checkout(branch)
@@ -346,7 +354,11 @@ def run(*args)
       '-Xtruffle.graal.warn_unless=false'
     ]
 
-    { '--asm' => '--graal', '--igv' => '--graal' }.each_pair do |arg, dep|
+    {
+        '--asm' => '--graal',
+        '--igv' => '--graal',
+        '--sulong' => '--graal'
+    }.each_pair do |arg, dep|
       args.unshift dep if args.include?(arg)
     end
 
@@ -360,6 +372,17 @@ def run(*args)
       jruby_args << Utilities.find_graal_js
     end
 
+    if args.delete('--sulong')
+      dir = Utilities.find_sulong_dir
+      jruby_args << '-J-classpath'
+      jruby_args << File.join(dir, 'lib', '*')
+      jruby_args << '-J-classpath'
+      jruby_args << File.join(dir, 'build', 'sulong.jar')
+      jruby_args << '-J-classpath'
+      jruby_args << File.join(dir, '..', 'graal-core', 'mxbuild', 'graal', 'com.oracle.nfi', 'bin')
+      jruby_args << '-J-XX:-UseJVMCIClassLoader'
+    end
+
     if args.delete('--asm')
       jruby_args += %w[-J-XX:+UnlockDiagnosticVMOptions -J-XX:CompileCommand=print,*::callRoot]
     end

diff --git a/truffle/src/main/java/org/jruby/truffle/RubyLanguage.java b/truffle/src/main/java/org/jruby/truffle/RubyLanguage.java
@@ -46,6 +46,9 @@ public class RubyLanguage extends TruffleLanguage<RubyContext> {
     public static final String MIME_TYPE = "application/x-ruby";
     public static final String EXTENSION = ".rb";
 
+    public static final String CEXT_MIME_TYPE = "application/x-sulong-library";
+    public static final String CEXT_EXTENSION = ".su";
+
     private RubyLanguage() {
     }
 

diff --git a/truffle/src/main/java/org/jruby/truffle/core/kernel/KernelNodes.java b/truffle/src/main/java/org/jruby/truffle/core/kernel/KernelNodes.java
@@ -1584,6 +1584,8 @@ public RubyNode coerceFeatureToPath(RubyNode feature) {
 
         @Specialization(guards = "isRubyString(featureString)")
         public boolean require(VirtualFrame frame, DynamicObject featureString, @Cached("create()") IndirectCallNode callNode) {
+            // We transfer because we want the virtual frame when requring but can't compile that code
+
             CompilerDirectives.transferToInterpreter();
 
             final String feature = featureString.toString();

diff --git a/truffle/src/main/java/org/jruby/truffle/language/loader/FeatureLoader.java b/truffle/src/main/java/org/jruby/truffle/language/loader/FeatureLoader.java
@@ -9,8 +9,16 @@
  */
 package org.jruby.truffle.language.loader;
 
+import com.oracle.truffle.api.CallTarget;
 import com.oracle.truffle.api.frame.VirtualFrame;
+import com.oracle.truffle.api.interop.ArityException;
+import com.oracle.truffle.api.interop.ForeignAccess;
+import com.oracle.truffle.api.interop.Message;
+import com.oracle.truffle.api.interop.TruffleObject;
+import com.oracle.truffle.api.interop.UnsupportedMessageException;
+import com.oracle.truffle.api.interop.UnsupportedTypeException;
 import com.oracle.truffle.api.nodes.IndirectCallNode;
+import com.oracle.truffle.api.nodes.Node;
 import com.oracle.truffle.api.object.DynamicObject;
 import com.oracle.truffle.api.source.Source;
 import org.jcodings.specific.UTF8Encoding;
@@ -78,6 +86,12 @@ private String findFeature(String feature) {
     }
 
     private String findFeatureWithAndWithoutExtension(String path) {
+        final String asCExt = findFeatureWithExactPath(path + RubyLanguage.CEXT_EXTENSION);
+
+        if (asCExt != null) {
+            return asCExt;
+        }
+
         final String withExtension = findFeatureWithExactPath(path + RubyLanguage.EXTENSION);
 
         if (withExtension != null) {
@@ -158,24 +172,68 @@ public Boolean block() throws InterruptedException {
                 return false;
             }
 
-            final DynamicObject pathString = StringOperations.createString(context,
-                    StringOperations.encodeRope(expandedPath, UTF8Encoding.INSTANCE));
+            final String mimeType = source.getMimeType();
+
+            switch (mimeType) {
+                case RubyLanguage.MIME_TYPE: {
+                    final RubyRootNode rootNode = context.getCodeLoader().parse(
+                            source,
+                            UTF8Encoding.INSTANCE,
+                            ParserContext.TOP_LEVEL,
+                            null,
+                            true,
+                            callNode);
+
+                    final CodeLoader.DeferredCall deferredCall = context.getCodeLoader().prepareExecute(
+                            ParserContext.TOP_LEVEL,
+                            DeclarationContext.TOP_LEVEL,
+                            rootNode, null,
+                            context.getCoreLibrary().getMainObject());
+
+                    deferredCall.call(frame, callNode);
+                } break;
+
+                case RubyLanguage.CEXT_MIME_TYPE: {
+                    final CallTarget callTarget;
+
+                    try {
+                        callTarget = context.getEnv().parse(source);
+                    } catch (IOException e) {
+                        throw new RuntimeException(e);
+                    }
+
+                    callNode.call(frame, callTarget, new Object[]{});
+
+                    final Object initFunction = context.getEnv().importSymbol("@Init_" + getBaseName(expandedPath));
+
+                    if (!(initFunction instanceof TruffleObject)) {
+                        throw new UnsupportedOperationException();
+                    }
 
-            final RubyRootNode rootNode = context.getCodeLoader().parse(
-                    source,
-                    UTF8Encoding.INSTANCE,
-                    ParserContext.TOP_LEVEL,
-                    null,
-                    true,
-                    callNode);
+                    final TruffleObject initFunctionObject = (TruffleObject) initFunction;
 
-            final CodeLoader.DeferredCall deferredCall = context.getCodeLoader().prepareExecute(
-                    ParserContext.TOP_LEVEL,
-                    DeclarationContext.TOP_LEVEL,
-                    rootNode, null,
-                    context.getCoreLibrary().getMainObject());
+                    final Node isExecutableNode = Message.IS_EXECUTABLE.createNode();
 
-            deferredCall.call(frame, callNode);
+                    if (!ForeignAccess.sendIsExecutable(isExecutableNode, frame, initFunctionObject)) {
+                        throw new UnsupportedOperationException();
+                    }
+
+                    final Node executeNode = Message.createExecute(0).createNode();
+
+                    try {
+                        ForeignAccess.sendExecute(executeNode, frame, initFunctionObject);
+                    } catch (UnsupportedTypeException | ArityException | UnsupportedMessageException e) {
+                        throw new RuntimeException(e);
+                    }
+                } break;
+
+                default:
+                    throw new RaiseException(
+                            context.getCoreLibrary().internalError("unknown language " + expandedPath, callNode));
+            }
+
+            final DynamicObject pathString = StringOperations.createString(context,
+                    StringOperations.encodeRope(expandedPath, UTF8Encoding.INSTANCE));
 
             addToLoadedFeatures(pathString);
 
@@ -186,6 +244,18 @@ public Boolean block() throws InterruptedException {
 
     }
 
+    private String getBaseName(String path) {
+        final String name = new File(path).getName();
+
+        final int firstDot = name.indexOf('.');
+
+        if (firstDot == -1) {
+            return name;
+        } else {
+            return name.substring(0, firstDot);
+        }
+    }
+
     // TODO (pitr-ch 16-Mar-2016): this protects the $LOADED_FEATURES only in this class,
     // it can still be accessed and modified (rare) by Ruby code which may cause issues
     private final Object loadedFeaturesLock = new Object();

diff --git a/truffle/src/main/ruby/core/truffle/truffle.rb b/truffle/src/main/ruby/core/truffle/truffle.rb
@@ -21,6 +21,10 @@ def version
     def graal?
       Primitive.graal?
     end
+
+    def cext?
+      Interop.mime_type_supported?('application/x-sulong-library')
+    end
 
     # Tests if this VM is a SubstrateVM build.
     def substrate?
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		org.jruby.util.RubyFiletypeDetector
		org.jruby.util.RubyFileTypeDetector
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		src: ext/minimum/*.c
		out: lib/minimum/minimum.su