Merge branch 'master' into truffle-head

jruby · Jun 30, 2016 · fbb6893 · fbb6893
2 parents db204d7 + f2e250e
commit fbb6893
Showing 17 changed files with 1,544 additions and 60 deletions.
diff --git a/core/src/main/java/org/jruby/RubyKernel.java b/core/src/main/java/org/jruby/RubyKernel.java
@@ -1146,7 +1146,7 @@ private static IRubyObject rbThrowInternal(ThreadContext context, IRubyObject ta
 
         final RubyThread currentThread = context.getThread();
         if (currentThread != runtime.getThreadService().getMainThread()) {
-            message += " in thread 0x" + Integer.toHexString(RubyInteger.fix2int(currentThread.id()));
+            message += " in thread 0x" + Integer.toHexString(System.identityHashCode(currentThread.getNativeThread()));
         }
         throw uncaughtThrow(runtime, tag, value, runtime.newString(message));
     }

diff --git a/lib/ruby/truffle/mri/unicode_normalize.rb b/lib/ruby/truffle/mri/unicode_normalize.rb
@@ -0,0 +1 @@
+require_relative '../stdlib/unicode_normalize'
diff --git a/lib/ruby/truffle/mri/unicode_normalize/normalize.rb b/lib/ruby/truffle/mri/unicode_normalize/normalize.rb
@@ -0,0 +1 @@
+require_relative '../../stdlib/unicode_normalize/' + File.basename(__FILE__)
diff --git a/lib/ruby/truffle/mri/unicode_normalize/tables.rb b/lib/ruby/truffle/mri/unicode_normalize/tables.rb
@@ -0,0 +1 @@
+require_relative '../../stdlib/unicode_normalize/' + File.basename(__FILE__)
diff --git a/lib/ruby/truffle/stdlib/unicode_normalize.rb b/lib/ruby/truffle/stdlib/unicode_normalize.rb
@@ -0,0 +1,77 @@
+# coding: utf-8
+
+# Copyright Ayumu Nojima (野島 歩) and Martin J. Dürst (duerst@it.aoyama.ac.jp)
+
+# additions to class String for Unicode normalization
+class String
+  # === Unicode Normalization
+  #
+  # :call-seq:
+  #    str.unicode_normalize(form=:nfc)
+  #
+  # Returns a normalized form of +str+, using Unicode normalizations
+  # NFC, NFD, NFKC, or NFKD. The normalization form used is determined
+  # by +form+, which is any of the four values :nfc, :nfd, :nfkc, or :nfkd.
+  # The default is :nfc.
+  #
+  # If the string is not in a Unicode Encoding, then an Exception is raised.
+  # In this context, 'Unicode Encoding' means any of  UTF-8, UTF-16BE/LE,
+  # and UTF-32BE/LE, as well as GB18030, UCS_2BE, and UCS_4BE. Anything
+  # else than UTF-8 is implemented by converting to UTF-8,
+  # which makes it slower than UTF-8.
+  #
+  # _Examples_
+  #
+  #   "a\u0300".unicode_normalize        #=> 'à' (same as "\u00E0")
+  #   "a\u0300".unicode_normalize(:nfc)  #=> 'à' (same as "\u00E0")
+  #   "\u00E0".unicode_normalize(:nfd)   #=> 'à' (same as "a\u0300")
+  #   "\xE0".force_encoding('ISO-8859-1').unicode_normalize(:nfd)
+  #                                      #=> Encoding::CompatibilityError raised
+  #
+  def unicode_normalize(form = :nfc)
+    require 'unicode_normalize/normalize.rb' unless defined? UnicodeNormalize
+    ## The following line can be uncommented to avoid repeated checking for
+    ## UnicodeNormalize. However, tests didn't show any noticeable speedup
+    ## when doing this. This comment also applies to the commented out lines
+    ## in String#unicode_normalize! and String#unicode_normalized?.
+    # String.send(:define_method, :unicode_normalize, ->(form = :nfc) { UnicodeNormalize.normalize(self, form) } )
+    UnicodeNormalize.normalize(self, form)
+  end
+
+  # :call-seq:
+  #    str.unicode_normalize!(form=:nfc)
+  #
+  # Destructive version of String#unicode_normalize, doing Unicode
+  # normalization in place.
+  #
+  def unicode_normalize!(form = :nfc)
+    require 'unicode_normalize/normalize.rb' unless defined? UnicodeNormalize
+    # String.send(:define_method, :unicode_normalize!, ->(form = :nfc) { replace(unicode_normalize(form)) } )
+    replace(unicode_normalize(form))
+  end
+
+  # :call-seq:
+  #    str.unicode_normalized?(form=:nfc)
+  #
+  # Checks whether +str+ is in Unicode normalization form +form+,
+  # which is any of the four values :nfc, :nfd, :nfkc, or :nfkd.
+  # The default is :nfc.
+  #
+  # If the string is not in a Unicode Encoding, then an Exception is raised.
+  # For details, see String#unicode_normalize.
+  #
+  # _Examples_
+  #
+  #   "a\u0300".unicode_normalized?        #=> false
+  #   "a\u0300".unicode_normalized?(:nfd)  #=> true
+  #   "\u00E0".unicode_normalized?         #=> true
+  #   "\u00E0".unicode_normalized?(:nfd)   #=> false
+  #   "\xE0".force_encoding('ISO-8859-1').unicode_normalized?
+  #                                        #=> Encoding::CompatibilityError raised
+  #
+  def unicode_normalized?(form = :nfc)
+    require 'unicode_normalize/normalize.rb' unless defined? UnicodeNormalize
+    # String.send(:define_method, :unicode_normalized?, ->(form = :nfc) { UnicodeNormalize.normalized?(self, form) } )
+    UnicodeNormalize.normalized?(self, form)
+  end
+end
diff --git a/lib/ruby/truffle/stdlib/unicode_normalize/normalize.rb b/lib/ruby/truffle/stdlib/unicode_normalize/normalize.rb
@@ -0,0 +1,168 @@
+# coding: utf-8
+
+# Copyright Ayumu Nojima (野島 歩) and Martin J. Dürst (duerst@it.aoyama.ac.jp)
+
+require 'unicode_normalize/tables.rb'
+
+
+module UnicodeNormalize
+  ## Constant for max hash capacity to avoid DoS attack
+  MAX_HASH_LENGTH = 18000 # enough for all test cases, otherwise tests get slow
+
+  ## Regular Expressions and Hash Constants
+  REGEXP_D = Regexp.compile(REGEXP_D_STRING, Regexp::EXTENDED)
+  REGEXP_C = Regexp.compile(REGEXP_C_STRING, Regexp::EXTENDED)
+  REGEXP_K = Regexp.compile(REGEXP_K_STRING, Regexp::EXTENDED)
+  NF_HASH_D = Hash.new do |hash, key|
+    hash.shift if hash.length>MAX_HASH_LENGTH # prevent DoS attack
+    hash[key] = nfd_one(key)
+  end
+  NF_HASH_C = Hash.new do |hash, key|
+    hash.shift if hash.length>MAX_HASH_LENGTH # prevent DoS attack
+    hash[key] = nfc_one(key)
+  end
+  NF_HASH_K = Hash.new do |hash, key|
+    hash.shift if hash.length>MAX_HASH_LENGTH # prevent DoS attack
+    hash[key] = nfkd_one(key)
+  end
+
+  ## Constants For Hangul
+  # for details such as the meaning of the identifiers below, please see
+  # http://www.unicode.org/versions/Unicode7.0.0/ch03.pdf, pp. 144/145
+  SBASE = 0xAC00
+  LBASE = 0x1100
+  VBASE = 0x1161
+  TBASE = 0x11A7
+  LCOUNT = 19
+  VCOUNT = 21
+  TCOUNT = 28
+  NCOUNT = VCOUNT * TCOUNT
+  SCOUNT = LCOUNT * NCOUNT
+
+  # Unicode-based encodings (except UTF-8)
+  UNICODE_ENCODINGS = [Encoding::UTF_16BE, Encoding::UTF_16LE, Encoding::UTF_32BE, Encoding::UTF_32LE,
+                       Encoding::GB18030, Encoding::UCS_2BE, Encoding::UCS_4BE]
+
+  ## Hangul Algorithm
+  def self.hangul_decomp_one(target)
+    syllable_index = target.ord - SBASE
+    return target if syllable_index < 0 || syllable_index >= SCOUNT
+    l = LBASE + syllable_index / NCOUNT
+    v = VBASE + (syllable_index % NCOUNT) / TCOUNT
+    t = TBASE + syllable_index % TCOUNT
+    (t==TBASE ? [l, v] : [l, v, t]).pack('U*') + target[1..-1]
+  end
+
+  def self.hangul_comp_one(string)
+    length = string.length
+    if length>1 and 0 <= (lead =string[0].ord-LBASE) and lead  < LCOUNT and
+        0 <= (vowel=string[1].ord-VBASE) and vowel < VCOUNT
+      lead_vowel = SBASE + (lead * VCOUNT + vowel) * TCOUNT
+      if length>2 and 0 <= (trail=string[2].ord-TBASE) and trail < TCOUNT
+        (lead_vowel + trail).chr(Encoding::UTF_8) + string[3..-1]
+      else
+        lead_vowel.chr(Encoding::UTF_8) + string[2..-1]
+      end
+    else
+      string
+    end
+  end
+
+  ## Canonical Ordering
+  def self.canonical_ordering_one(string)
+    sorting = string.each_char.collect { |c| [c, CLASS_TABLE[c]] }
+    (sorting.length-2).downto(0) do |i| # almost, but not exactly bubble sort
+      (0..i).each do |j|
+        later_class = sorting[j+1].last
+        if 0<later_class and later_class<sorting[j].last
+          sorting[j], sorting[j+1] = sorting[j+1], sorting[j]
+        end
+      end
+    end
+    return sorting.collect(&:first).join('')
+  end
+
+  ## Normalization Forms for Patterns (not whole Strings)
+  def self.nfd_one(string)
+    string = string.chars.map! {|c| DECOMPOSITION_TABLE[c] || c}.join('')
+    canonical_ordering_one(hangul_decomp_one(string))
+  end
+
+  def self.nfkd_one(string)
+    string.chars.map! {|c| KOMPATIBLE_TABLE[c] || c}.join('')
+  end
+
+  def self.nfc_one(string)
+    nfd_string = nfd_one string
+    start = nfd_string[0]
+    last_class = CLASS_TABLE[start]-1
+    accents = ''
+    nfd_string[1..-1].each_char do |accent|
+      accent_class = CLASS_TABLE[accent]
+      if last_class<accent_class and composite = COMPOSITION_TABLE[start+accent]
+        start = composite
+      else
+        accents << accent
+        last_class = accent_class
+      end
+    end
+    hangul_comp_one(start+accents)
+  end
+
+  def self.normalize(string, form = :nfc)
+    encoding = string.encoding
+    case encoding
+      when Encoding::UTF_8
+        case form
+          when :nfc then
+            string.gsub REGEXP_C, NF_HASH_C
+          when :nfd then
+            string.gsub REGEXP_D, NF_HASH_D
+          when :nfkc then
+            string.gsub(REGEXP_K, NF_HASH_K).gsub REGEXP_C, NF_HASH_C
+          when :nfkd then
+            string.gsub(REGEXP_K, NF_HASH_K).gsub REGEXP_D, NF_HASH_D
+          else
+            raise ArgumentError, "Invalid normalization form #{form}."
+        end
+      when Encoding::US_ASCII
+        string
+      when *UNICODE_ENCODINGS
+        normalize(string.encode(Encoding::UTF_8), form).encode(encoding)
+      else
+        raise Encoding::CompatibilityError, "Unicode Normalization not appropriate for #{encoding}"
+    end
+  end
+
+  def self.normalized?(string, form = :nfc)
+    encoding = string.encoding
+    case encoding
+      when Encoding::UTF_8
+        case form
+          when :nfc then
+            string.scan REGEXP_C do |match|
+              return false  if NF_HASH_C[match] != match
+            end
+            true
+          when :nfd then
+            string.scan REGEXP_D do |match|
+              return false  if NF_HASH_D[match] != match
+            end
+            true
+          when :nfkc then
+            normalized?(string, :nfc) and string !~ REGEXP_K
+          when :nfkd then
+            normalized?(string, :nfd) and string !~ REGEXP_K
+          else
+            raise ArgumentError, "Invalid normalization form #{form}."
+        end
+      when Encoding::US_ASCII
+        true
+      when *UNICODE_ENCODINGS
+        normalized? string.encode(Encoding::UTF_8), form
+      else
+        raise Encoding::CompatibilityError, "Unicode Normalization not appropriate for #{encoding}"
+    end
+  end
+
+end # module
diff --git a/lib/ruby/truffle/stdlib/unicode_normalize/tables.rb b/lib/ruby/truffle/stdlib/unicode_normalize/tables.rb
diff --git a/mx.jruby/mx_jruby.py b/mx.jruby/mx_jruby.py
@@ -100,24 +100,28 @@ def build(self):
         truffle_commit = truffle.vc.parent(truffle.dir)
         maven_version_arg = '-Dtruffle.version=' + truffle_commit
         maven_repo_arg = '-Dmaven.repo.local=' + mavenDir
-
+        
         mx.run_mx(['maven-install', '--repo', mavenDir, '--only', 'TRUFFLE_API,TRUFFLE_DEBUG,TRUFFLE_DSL_PROCESSOR,TRUFFLE_TCK'], suite=truffle)
 
         open(os.path.join(rubyDir, 'VERSION'), 'w').write('graal-vm\n')
 
         # Build jruby-truffle
 
-        mx.run_maven([ '--version', maven_repo_arg], nonZeroIsFatal=False, cwd=rubyDir)
+        env = os.environ.copy()
+        env['JRUBY_BUILD_MORE_QUIET'] = 'true'
+
+        mx.run_maven(['-q', '--version', maven_repo_arg], nonZeroIsFatal=False, cwd=rubyDir, env=env)
 
         mx.log('Building without tests')
 
-        mx.run_maven(['-DskipTests', maven_version_arg, maven_repo_arg], cwd=rubyDir)
+        mx.run_maven(['-q', '-DskipTests', maven_version_arg, maven_repo_arg], cwd=rubyDir, env=env)
 
         mx.log('Building complete version')
 
-        mx.run_maven(['-Pcomplete', '-DskipTests', maven_version_arg, maven_repo_arg], cwd=rubyDir)
+        mx.run_maven(['-q', '-Pcomplete', '-DskipTests', maven_version_arg, maven_repo_arg], cwd=rubyDir, env=env)
         mx.run(['zip', '-d', 'maven/jruby-complete/target/jruby-complete-graal-vm.jar', 'META-INF/jruby.home/lib/*'], cwd=rubyDir)
         mx.run(['bin/jruby', 'bin/gem', 'install', 'bundler', '-v', '1.10.6'], cwd=rubyDir)
+
         mx.log('...finished build of {}'.format(self.subject))
 
     def clean(self, forBuild=False):
@@ -126,6 +130,82 @@ def clean(self, forBuild=False):
         rubyDir = _suite.dir
         mx.run_maven(['-q', 'clean'], nonZeroIsFatal=False, cwd=rubyDir)
 
+class LicensesProject(mx.Project):
+    def __init__(self, suite, name, deps, workingSets, theLicense, **args):
+        mx.Project.__init__(self, suite, name, "", [], deps, workingSets, _suite.dir, theLicense)
+        self.javaCompliance = "1.7"
+        self.build = hasattr(args, 'build')
+        self.prefix = args['prefix']
+
+    def source_dirs(self):
+        return []
+
+    def output_dir(self, relative=False):
+        dir = os.path.join(_suite.dir, self.prefix)
+        return dir.rstrip('/')
+
+    def source_gen_dir(self):
+        return None
+
+    def getOutput(self, replaceVar=False):
+        return os.path.join(_suite.dir, "target")
+
+    def getResults(self, replaceVar=False):
+        return mx.Project.getResults(self, replaceVar=replaceVar)
+
+    def getBuildTask(self, args):
+        return LicensesBuildTask(self, args, None, None)
+
+    def isJavaProject(self):
+        return True
+
+    def archive_prefix(self):
+        return self.prefix
+
+    def annotation_processors(self):
+        return []
+
+    def find_classes_with_matching_source_line(self, pkgRoot, function, includeInnerClasses=False):
+        return dict()
+
+class LicensesBuildTask(mx.BuildTask):
+    def __init__(self, project, args, vmbuild, vm):
+        mx.BuildTask.__init__(self, project, args, 1)
+        self.vm = vm
+        self.vmbuild = vmbuild
+
+    def __str__(self):
+        return 'Building licences for {}'.format(self.subject)
+
+    def needsBuild(self, newestInput):
+        return (True, 'Let us re-build everytime')
+
+    def newestOutput(self):
+        return None
+
+    def build(self):
+        if not self.subject.build:
+            mx.log("...skip build of {}".format(self.subject))
+            return
+        mx.log('...perform build of {}'.format(self.subject))
+
+        rubyDir = _suite.dir
+        licenses_dir = os.path.join(rubyDir, 'licenses')
+        if not os.path.exists(licenses_dir):
+            os.mkdir(licenses_dir)
+        for f in ['BSDL', 'COPYING', 'LICENSE.RUBY']:
+            shutil.copyfile(os.path.join(rubyDir, f), os.path.join(licenses_dir, f))
+
+        mx.log('...finished build of {}'.format(self.subject))
+
+    def clean(self, forBuild=False):
+        if forBuild:
+            return
+        rubyDir = _suite.dir
+        licenses_dir = os.path.join(rubyDir, 'licenses')
+        if os.path.exists(licenses_dir):
+            shutil.rmtree(licenses_dir)
+
 class RubyBenchmarkSuite(mx_benchmark.BenchmarkSuite):
     def group(self):
         return 'Graal'

diff --git a/mx.jruby/suite.py b/mx.jruby/suite.py
@@ -50,8 +50,6 @@
             "dependencies": [
                 "truffle:TRUFFLE_API",
                 "truffle:TRUFFLE_DEBUG",
-                "truffle:TRUFFLE_DSL_PROCESSOR",
-                "truffle:TRUFFLE_TCK",
             ],
         },
         "jruby-lib-jni": {
@@ -61,8 +59,16 @@
             "dependencies": [
                 "truffle:TRUFFLE_API",
                 "truffle:TRUFFLE_DEBUG",
-                "truffle:TRUFFLE_DSL_PROCESSOR",
-                "truffle:TRUFFLE_TCK",
+            ],
+        },
+        "jruby-licences": {
+            "subDir": "licenses",
+            "class": "LicensesProject",
+            "build": "true",
+            "prefix": "licenses",
+            "dependencies": [
+                "truffle:TRUFFLE_API",
+                "truffle:TRUFFLE_DEBUG",
             ],
         },
     },
@@ -89,8 +95,6 @@
                 "RUBY-ZIP",
                 "truffle:TRUFFLE_API",
                 "truffle:TRUFFLE_DEBUG",
- 		"truffle:TRUFFLE_DSL_PROCESSOR",
-                "truffle:TRUFFLE_TCK",
             ],
             "description": "JRuby+Truffle",
             "license": "EPL"
@@ -99,15 +103,14 @@
             "dependencies": [
                 "jruby-ruby",
                 "jruby-lib-jni",
+                "jruby-licences",
             ],
             "exclude": [
                 "truffle:JLINE",
             ],
             "distDependencies": [
                 "truffle:TRUFFLE_API",
                 "truffle:TRUFFLE_DEBUG",
-		"truffle:TRUFFLE_DSL_PROCESSOR",
-                "truffle:TRUFFLE_TCK",
             ],
             "description": "JRuby+Truffle Native Libs",
             "license": "EPL"

diff --git a/spec/truffle/tags/core/array/delete_tags.txt b/spec/truffle/tags/core/array/delete_tags.txt
diff --git a/spec/truffle/tags/core/string/sub_tags.txt b/spec/truffle/tags/core/string/sub_tags.txt
diff --git a/spec/truffle/tags/core/string/unicode_normalize_tags.txt b/spec/truffle/tags/core/string/unicode_normalize_tags.txt
diff --git a/spec/truffle/tags/core/string/unicode_normalized_tags.txt b/spec/truffle/tags/core/string/unicode_normalized_tags.txt
diff --git a/truffle/src/main/java/org/jruby/truffle/core/array/ArrayNodes.java b/truffle/src/main/java/org/jruby/truffle/core/array/ArrayNodes.java
@@ -695,8 +695,9 @@ public DynamicObject concat(DynamicObject array, DynamicObject other) {
 
     }
 
-    @CoreMethod(names = "delete", required = 1)
-    public abstract static class DeleteNode extends ArrayCoreMethodNode {
+    @CoreMethod(names = "delete", required = 1, needsBlock = true)
+    @ImportStatic(ArrayGuards.class)
+    public abstract static class DeleteNode extends YieldingCoreMethodNode {
 
         @Child private KernelNodes.SameOrEqualNode equalNode;
         @Child private IsFrozenNode isFrozenNode;
@@ -707,16 +708,22 @@ public DeleteNode(RubyContext context, SourceSection sourceSection) {
         }
 
         @Specialization(guards = "isNullArray(array)")
-        public Object deleteNull(VirtualFrame frame, DynamicObject array, Object value) {
+        public Object deleteNull(VirtualFrame frame, DynamicObject array, Object value, NotProvided block) {
             return nil();
         }
 
+        @Specialization(guards = "isNullArray(array)")
+        public Object deleteNull(VirtualFrame frame, DynamicObject array, Object value, DynamicObject block) {
+            return yield(frame, block, value);
+        }
+
         @Specialization(guards = "strategy.matches(array)", limit = "ARRAY_STRATEGIES")
-        public Object delete(VirtualFrame frame, DynamicObject array, Object value,
+        public Object delete(VirtualFrame frame, DynamicObject array, Object value, Object maybeBlock,
                 @Cached("of(array)") ArrayStrategy strategy) {
             final ArrayMirror store = strategy.newMirror(array);
 
             Object found = nil();
+            boolean isFound = false;
 
             int i = 0;
             int n = 0;
@@ -726,6 +733,7 @@ public Object delete(VirtualFrame frame, DynamicObject array, Object value,
                 if (equalNode.executeSameOrEqual(frame, stored, value)) {
                     checkFrozen(array);
                     found = stored;
+                    isFound = true;
                     continue;
                 }
 
@@ -739,6 +747,15 @@ public Object delete(VirtualFrame frame, DynamicObject array, Object value,
             if (i != n) {
                 setStoreAndSize(array, store.getArray(), i);
             }
+
+            if(!isFound){
+                if (maybeBlock == NotProvided.INSTANCE) {
+                    return nil();
+                } else {
+                    return yield(frame, (DynamicObject) maybeBlock, value);
+                }
+            }
+
             return found;
         }
 

diff --git a/truffle/src/main/ruby/core/hash.rb b/truffle/src/main/ruby/core/hash.rb
@@ -37,7 +37,12 @@ class Hash
 
   def self.contains_all_internal(one, two)
     one.all? do |key, value|
-      two.has_key?(key) && (value.equal?(two[key]) || value == two[key])
+      if two.has_key?(key)
+        two_value = two[key]
+        value.equal?(two_value) || value == two_value
+      else
+        false
+      end
     end
   end
 

diff --git a/truffle/src/main/ruby/core/post.rb b/truffle/src/main/ruby/core/post.rb
@@ -93,3 +93,9 @@ def include(*modules)
   end
 
 end
+
+# TODO BJF 29-Jun-2016 Normally loaded by a prelude file, review where to load this
+begin
+  require 'unicode_normalize'
+rescue LoadError
+end
diff --git a/truffle/src/main/ruby/core/string.rb b/truffle/src/main/ruby/core/string.rb
@@ -847,11 +847,15 @@ def sub!(pattern, replacement=undefined)
       if use_yield || hash
         Regexp.last_match = match
 
+        duped = dup
         if use_yield
           val = yield match.to_s
         else
           val = hash[match.to_s]
         end
+        if duped != self
+          raise RuntimeError, "string modified"
+        end
         untrusted = true if val.untrusted?
         val = val.to_s unless val.kind_of?(String)
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		require_relative '../../stdlib/unicode_normalize/' + File.basename(__FILE__)