Skip to content

Commit

Permalink
Showing 17 changed files with 1,544 additions and 60 deletions.
2 changes: 1 addition & 1 deletion core/src/main/java/org/jruby/RubyKernel.java
Original file line number Diff line number Diff line change
@@ -1146,7 +1146,7 @@ private static IRubyObject rbThrowInternal(ThreadContext context, IRubyObject ta

final RubyThread currentThread = context.getThread();
if (currentThread != runtime.getThreadService().getMainThread()) {
message += " in thread 0x" + Integer.toHexString(RubyInteger.fix2int(currentThread.id()));
message += " in thread 0x" + Integer.toHexString(System.identityHashCode(currentThread.getNativeThread()));
}
throw uncaughtThrow(runtime, tag, value, runtime.newString(message));
}
1 change: 1 addition & 0 deletions lib/ruby/truffle/mri/unicode_normalize.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
require_relative '../stdlib/unicode_normalize'
1 change: 1 addition & 0 deletions lib/ruby/truffle/mri/unicode_normalize/normalize.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
require_relative '../../stdlib/unicode_normalize/' + File.basename(__FILE__)
1 change: 1 addition & 0 deletions lib/ruby/truffle/mri/unicode_normalize/tables.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
require_relative '../../stdlib/unicode_normalize/' + File.basename(__FILE__)
77 changes: 77 additions & 0 deletions lib/ruby/truffle/stdlib/unicode_normalize.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
# coding: utf-8

# Copyright Ayumu Nojima (野島 歩) and Martin J. Dürst (duerst@it.aoyama.ac.jp)

# additions to class String for Unicode normalization
class String
# === Unicode Normalization
#
# :call-seq:
# str.unicode_normalize(form=:nfc)
#
# Returns a normalized form of +str+, using Unicode normalizations
# NFC, NFD, NFKC, or NFKD. The normalization form used is determined
# by +form+, which is any of the four values :nfc, :nfd, :nfkc, or :nfkd.
# The default is :nfc.
#
# If the string is not in a Unicode Encoding, then an Exception is raised.
# In this context, 'Unicode Encoding' means any of UTF-8, UTF-16BE/LE,
# and UTF-32BE/LE, as well as GB18030, UCS_2BE, and UCS_4BE. Anything
# else than UTF-8 is implemented by converting to UTF-8,
# which makes it slower than UTF-8.
#
# _Examples_
#
# "a\u0300".unicode_normalize #=> 'à' (same as "\u00E0")
# "a\u0300".unicode_normalize(:nfc) #=> 'à' (same as "\u00E0")
# "\u00E0".unicode_normalize(:nfd) #=> 'à' (same as "a\u0300")
# "\xE0".force_encoding('ISO-8859-1').unicode_normalize(:nfd)
# #=> Encoding::CompatibilityError raised
#
def unicode_normalize(form = :nfc)
require 'unicode_normalize/normalize.rb' unless defined? UnicodeNormalize
## The following line can be uncommented to avoid repeated checking for
## UnicodeNormalize. However, tests didn't show any noticeable speedup
## when doing this. This comment also applies to the commented out lines
## in String#unicode_normalize! and String#unicode_normalized?.
# String.send(:define_method, :unicode_normalize, ->(form = :nfc) { UnicodeNormalize.normalize(self, form) } )
UnicodeNormalize.normalize(self, form)
end

# :call-seq:
# str.unicode_normalize!(form=:nfc)
#
# Destructive version of String#unicode_normalize, doing Unicode
# normalization in place.
#
def unicode_normalize!(form = :nfc)
require 'unicode_normalize/normalize.rb' unless defined? UnicodeNormalize
# String.send(:define_method, :unicode_normalize!, ->(form = :nfc) { replace(unicode_normalize(form)) } )
replace(unicode_normalize(form))
end

# :call-seq:
# str.unicode_normalized?(form=:nfc)
#
# Checks whether +str+ is in Unicode normalization form +form+,
# which is any of the four values :nfc, :nfd, :nfkc, or :nfkd.
# The default is :nfc.
#
# If the string is not in a Unicode Encoding, then an Exception is raised.
# For details, see String#unicode_normalize.
#
# _Examples_
#
# "a\u0300".unicode_normalized? #=> false
# "a\u0300".unicode_normalized?(:nfd) #=> true
# "\u00E0".unicode_normalized? #=> true
# "\u00E0".unicode_normalized?(:nfd) #=> false
# "\xE0".force_encoding('ISO-8859-1').unicode_normalized?
# #=> Encoding::CompatibilityError raised
#
def unicode_normalized?(form = :nfc)
require 'unicode_normalize/normalize.rb' unless defined? UnicodeNormalize
# String.send(:define_method, :unicode_normalized?, ->(form = :nfc) { UnicodeNormalize.normalized?(self, form) } )
UnicodeNormalize.normalized?(self, form)
end
end
168 changes: 168 additions & 0 deletions lib/ruby/truffle/stdlib/unicode_normalize/normalize.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,168 @@
# coding: utf-8

# Copyright Ayumu Nojima (野島 歩) and Martin J. Dürst (duerst@it.aoyama.ac.jp)

require 'unicode_normalize/tables.rb'


module UnicodeNormalize
## Constant for max hash capacity to avoid DoS attack
MAX_HASH_LENGTH = 18000 # enough for all test cases, otherwise tests get slow

## Regular Expressions and Hash Constants
REGEXP_D = Regexp.compile(REGEXP_D_STRING, Regexp::EXTENDED)
REGEXP_C = Regexp.compile(REGEXP_C_STRING, Regexp::EXTENDED)
REGEXP_K = Regexp.compile(REGEXP_K_STRING, Regexp::EXTENDED)
NF_HASH_D = Hash.new do |hash, key|
hash.shift if hash.length>MAX_HASH_LENGTH # prevent DoS attack
hash[key] = nfd_one(key)
end
NF_HASH_C = Hash.new do |hash, key|
hash.shift if hash.length>MAX_HASH_LENGTH # prevent DoS attack
hash[key] = nfc_one(key)
end
NF_HASH_K = Hash.new do |hash, key|
hash.shift if hash.length>MAX_HASH_LENGTH # prevent DoS attack
hash[key] = nfkd_one(key)
end

## Constants For Hangul
# for details such as the meaning of the identifiers below, please see
# http://www.unicode.org/versions/Unicode7.0.0/ch03.pdf, pp. 144/145
SBASE = 0xAC00
LBASE = 0x1100
VBASE = 0x1161
TBASE = 0x11A7
LCOUNT = 19
VCOUNT = 21
TCOUNT = 28
NCOUNT = VCOUNT * TCOUNT
SCOUNT = LCOUNT * NCOUNT

# Unicode-based encodings (except UTF-8)
UNICODE_ENCODINGS = [Encoding::UTF_16BE, Encoding::UTF_16LE, Encoding::UTF_32BE, Encoding::UTF_32LE,
Encoding::GB18030, Encoding::UCS_2BE, Encoding::UCS_4BE]

## Hangul Algorithm
def self.hangul_decomp_one(target)
syllable_index = target.ord - SBASE
return target if syllable_index < 0 || syllable_index >= SCOUNT
l = LBASE + syllable_index / NCOUNT
v = VBASE + (syllable_index % NCOUNT) / TCOUNT
t = TBASE + syllable_index % TCOUNT
(t==TBASE ? [l, v] : [l, v, t]).pack('U*') + target[1..-1]
end

def self.hangul_comp_one(string)
length = string.length
if length>1 and 0 <= (lead =string[0].ord-LBASE) and lead < LCOUNT and
0 <= (vowel=string[1].ord-VBASE) and vowel < VCOUNT
lead_vowel = SBASE + (lead * VCOUNT + vowel) * TCOUNT
if length>2 and 0 <= (trail=string[2].ord-TBASE) and trail < TCOUNT
(lead_vowel + trail).chr(Encoding::UTF_8) + string[3..-1]
else
lead_vowel.chr(Encoding::UTF_8) + string[2..-1]
end
else
string
end
end

## Canonical Ordering
def self.canonical_ordering_one(string)
sorting = string.each_char.collect { |c| [c, CLASS_TABLE[c]] }
(sorting.length-2).downto(0) do |i| # almost, but not exactly bubble sort
(0..i).each do |j|
later_class = sorting[j+1].last
if 0<later_class and later_class<sorting[j].last
sorting[j], sorting[j+1] = sorting[j+1], sorting[j]
end
end
end
return sorting.collect(&:first).join('')
end

## Normalization Forms for Patterns (not whole Strings)
def self.nfd_one(string)
string = string.chars.map! {|c| DECOMPOSITION_TABLE[c] || c}.join('')
canonical_ordering_one(hangul_decomp_one(string))
end

def self.nfkd_one(string)
string.chars.map! {|c| KOMPATIBLE_TABLE[c] || c}.join('')
end

def self.nfc_one(string)
nfd_string = nfd_one string
start = nfd_string[0]
last_class = CLASS_TABLE[start]-1
accents = ''
nfd_string[1..-1].each_char do |accent|
accent_class = CLASS_TABLE[accent]
if last_class<accent_class and composite = COMPOSITION_TABLE[start+accent]
start = composite
else
accents << accent
last_class = accent_class
end
end
hangul_comp_one(start+accents)
end

def self.normalize(string, form = :nfc)
encoding = string.encoding
case encoding
when Encoding::UTF_8
case form
when :nfc then
string.gsub REGEXP_C, NF_HASH_C
when :nfd then
string.gsub REGEXP_D, NF_HASH_D
when :nfkc then
string.gsub(REGEXP_K, NF_HASH_K).gsub REGEXP_C, NF_HASH_C
when :nfkd then
string.gsub(REGEXP_K, NF_HASH_K).gsub REGEXP_D, NF_HASH_D
else
raise ArgumentError, "Invalid normalization form #{form}."
end
when Encoding::US_ASCII
string
when *UNICODE_ENCODINGS
normalize(string.encode(Encoding::UTF_8), form).encode(encoding)
else
raise Encoding::CompatibilityError, "Unicode Normalization not appropriate for #{encoding}"
end
end

def self.normalized?(string, form = :nfc)
encoding = string.encoding
case encoding
when Encoding::UTF_8
case form
when :nfc then
string.scan REGEXP_C do |match|
return false if NF_HASH_C[match] != match
end
true
when :nfd then
string.scan REGEXP_D do |match|
return false if NF_HASH_D[match] != match
end
true
when :nfkc then
normalized?(string, :nfc) and string !~ REGEXP_K
when :nfkd then
normalized?(string, :nfd) and string !~ REGEXP_K
else
raise ArgumentError, "Invalid normalization form #{form}."
end
when Encoding::US_ASCII
true
when *UNICODE_ENCODINGS
normalized? string.encode(Encoding::UTF_8), form
else
raise Encoding::CompatibilityError, "Unicode Normalization not appropriate for #{encoding}"
end
end

end # module
1,163 changes: 1,163 additions & 0 deletions lib/ruby/truffle/stdlib/unicode_normalize/tables.rb

Large diffs are not rendered by default.

88 changes: 84 additions & 4 deletions mx.jruby/mx_jruby.py
Original file line number Diff line number Diff line change
@@ -100,24 +100,28 @@ def build(self):
truffle_commit = truffle.vc.parent(truffle.dir)
maven_version_arg = '-Dtruffle.version=' + truffle_commit
maven_repo_arg = '-Dmaven.repo.local=' + mavenDir

mx.run_mx(['maven-install', '--repo', mavenDir, '--only', 'TRUFFLE_API,TRUFFLE_DEBUG,TRUFFLE_DSL_PROCESSOR,TRUFFLE_TCK'], suite=truffle)

open(os.path.join(rubyDir, 'VERSION'), 'w').write('graal-vm\n')

# Build jruby-truffle

mx.run_maven([ '--version', maven_repo_arg], nonZeroIsFatal=False, cwd=rubyDir)
env = os.environ.copy()
env['JRUBY_BUILD_MORE_QUIET'] = 'true'

mx.run_maven(['-q', '--version', maven_repo_arg], nonZeroIsFatal=False, cwd=rubyDir, env=env)

mx.log('Building without tests')

mx.run_maven(['-DskipTests', maven_version_arg, maven_repo_arg], cwd=rubyDir)
mx.run_maven(['-q', '-DskipTests', maven_version_arg, maven_repo_arg], cwd=rubyDir, env=env)

mx.log('Building complete version')

mx.run_maven(['-Pcomplete', '-DskipTests', maven_version_arg, maven_repo_arg], cwd=rubyDir)
mx.run_maven(['-q', '-Pcomplete', '-DskipTests', maven_version_arg, maven_repo_arg], cwd=rubyDir, env=env)
mx.run(['zip', '-d', 'maven/jruby-complete/target/jruby-complete-graal-vm.jar', 'META-INF/jruby.home/lib/*'], cwd=rubyDir)
mx.run(['bin/jruby', 'bin/gem', 'install', 'bundler', '-v', '1.10.6'], cwd=rubyDir)

mx.log('...finished build of {}'.format(self.subject))

def clean(self, forBuild=False):
@@ -126,6 +130,82 @@ def clean(self, forBuild=False):
rubyDir = _suite.dir
mx.run_maven(['-q', 'clean'], nonZeroIsFatal=False, cwd=rubyDir)

class LicensesProject(mx.Project):
def __init__(self, suite, name, deps, workingSets, theLicense, **args):
mx.Project.__init__(self, suite, name, "", [], deps, workingSets, _suite.dir, theLicense)
self.javaCompliance = "1.7"
self.build = hasattr(args, 'build')
self.prefix = args['prefix']

def source_dirs(self):
return []

def output_dir(self, relative=False):
dir = os.path.join(_suite.dir, self.prefix)
return dir.rstrip('/')

def source_gen_dir(self):
return None

def getOutput(self, replaceVar=False):
return os.path.join(_suite.dir, "target")

def getResults(self, replaceVar=False):
return mx.Project.getResults(self, replaceVar=replaceVar)

def getBuildTask(self, args):
return LicensesBuildTask(self, args, None, None)

def isJavaProject(self):
return True

def archive_prefix(self):
return self.prefix

def annotation_processors(self):
return []

def find_classes_with_matching_source_line(self, pkgRoot, function, includeInnerClasses=False):
return dict()

class LicensesBuildTask(mx.BuildTask):
def __init__(self, project, args, vmbuild, vm):
mx.BuildTask.__init__(self, project, args, 1)
self.vm = vm
self.vmbuild = vmbuild

def __str__(self):
return 'Building licences for {}'.format(self.subject)

def needsBuild(self, newestInput):
return (True, 'Let us re-build everytime')

def newestOutput(self):
return None

def build(self):
if not self.subject.build:
mx.log("...skip build of {}".format(self.subject))
return
mx.log('...perform build of {}'.format(self.subject))

rubyDir = _suite.dir
licenses_dir = os.path.join(rubyDir, 'licenses')
if not os.path.exists(licenses_dir):
os.mkdir(licenses_dir)
for f in ['BSDL', 'COPYING', 'LICENSE.RUBY']:
shutil.copyfile(os.path.join(rubyDir, f), os.path.join(licenses_dir, f))

mx.log('...finished build of {}'.format(self.subject))

def clean(self, forBuild=False):
if forBuild:
return
rubyDir = _suite.dir
licenses_dir = os.path.join(rubyDir, 'licenses')
if os.path.exists(licenses_dir):
shutil.rmtree(licenses_dir)

class RubyBenchmarkSuite(mx_benchmark.BenchmarkSuite):
def group(self):
return 'Graal'
19 changes: 11 additions & 8 deletions mx.jruby/suite.py
Original file line number Diff line number Diff line change
@@ -50,8 +50,6 @@
"dependencies": [
"truffle:TRUFFLE_API",
"truffle:TRUFFLE_DEBUG",
"truffle:TRUFFLE_DSL_PROCESSOR",
"truffle:TRUFFLE_TCK",
],
},
"jruby-lib-jni": {
@@ -61,8 +59,16 @@
"dependencies": [
"truffle:TRUFFLE_API",
"truffle:TRUFFLE_DEBUG",
"truffle:TRUFFLE_DSL_PROCESSOR",
"truffle:TRUFFLE_TCK",
],
},
"jruby-licences": {
"subDir": "licenses",
"class": "LicensesProject",
"build": "true",
"prefix": "licenses",
"dependencies": [
"truffle:TRUFFLE_API",
"truffle:TRUFFLE_DEBUG",
],
},
},
@@ -89,8 +95,6 @@
"RUBY-ZIP",
"truffle:TRUFFLE_API",
"truffle:TRUFFLE_DEBUG",
"truffle:TRUFFLE_DSL_PROCESSOR",
"truffle:TRUFFLE_TCK",
],
"description": "JRuby+Truffle",
"license": "EPL"
@@ -99,15 +103,14 @@
"dependencies": [
"jruby-ruby",
"jruby-lib-jni",
"jruby-licences",
],
"exclude": [
"truffle:JLINE",
],
"distDependencies": [
"truffle:TRUFFLE_API",
"truffle:TRUFFLE_DEBUG",
"truffle:TRUFFLE_DSL_PROCESSOR",
"truffle:TRUFFLE_TCK",
],
"description": "JRuby+Truffle Native Libs",
"license": "EPL"
1 change: 0 additions & 1 deletion spec/truffle/tags/core/array/delete_tags.txt

This file was deleted.

1 change: 0 additions & 1 deletion spec/truffle/tags/core/string/sub_tags.txt

This file was deleted.

17 changes: 0 additions & 17 deletions spec/truffle/tags/core/string/unicode_normalize_tags.txt

This file was deleted.

23 changes: 0 additions & 23 deletions spec/truffle/tags/core/string/unicode_normalized_tags.txt

This file was deleted.

25 changes: 21 additions & 4 deletions truffle/src/main/java/org/jruby/truffle/core/array/ArrayNodes.java
Original file line number Diff line number Diff line change
@@ -695,8 +695,9 @@ public DynamicObject concat(DynamicObject array, DynamicObject other) {

}

@CoreMethod(names = "delete", required = 1)
public abstract static class DeleteNode extends ArrayCoreMethodNode {
@CoreMethod(names = "delete", required = 1, needsBlock = true)
@ImportStatic(ArrayGuards.class)
public abstract static class DeleteNode extends YieldingCoreMethodNode {

@Child private KernelNodes.SameOrEqualNode equalNode;
@Child private IsFrozenNode isFrozenNode;
@@ -707,16 +708,22 @@ public DeleteNode(RubyContext context, SourceSection sourceSection) {
}

@Specialization(guards = "isNullArray(array)")
public Object deleteNull(VirtualFrame frame, DynamicObject array, Object value) {
public Object deleteNull(VirtualFrame frame, DynamicObject array, Object value, NotProvided block) {
return nil();
}

@Specialization(guards = "isNullArray(array)")
public Object deleteNull(VirtualFrame frame, DynamicObject array, Object value, DynamicObject block) {
return yield(frame, block, value);
}

@Specialization(guards = "strategy.matches(array)", limit = "ARRAY_STRATEGIES")
public Object delete(VirtualFrame frame, DynamicObject array, Object value,
public Object delete(VirtualFrame frame, DynamicObject array, Object value, Object maybeBlock,
@Cached("of(array)") ArrayStrategy strategy) {
final ArrayMirror store = strategy.newMirror(array);

Object found = nil();
boolean isFound = false;

int i = 0;
int n = 0;
@@ -726,6 +733,7 @@ public Object delete(VirtualFrame frame, DynamicObject array, Object value,
if (equalNode.executeSameOrEqual(frame, stored, value)) {
checkFrozen(array);
found = stored;
isFound = true;
continue;
}

@@ -739,6 +747,15 @@ public Object delete(VirtualFrame frame, DynamicObject array, Object value,
if (i != n) {
setStoreAndSize(array, store.getArray(), i);
}

if(!isFound){
if (maybeBlock == NotProvided.INSTANCE) {
return nil();
} else {
return yield(frame, (DynamicObject) maybeBlock, value);
}
}

return found;
}

7 changes: 6 additions & 1 deletion truffle/src/main/ruby/core/hash.rb
Original file line number Diff line number Diff line change
@@ -37,7 +37,12 @@ class Hash

def self.contains_all_internal(one, two)
one.all? do |key, value|
two.has_key?(key) && (value.equal?(two[key]) || value == two[key])
if two.has_key?(key)
two_value = two[key]
value.equal?(two_value) || value == two_value
else
false
end
end
end

6 changes: 6 additions & 0 deletions truffle/src/main/ruby/core/post.rb
Original file line number Diff line number Diff line change
@@ -93,3 +93,9 @@ def include(*modules)
end

end

# TODO BJF 29-Jun-2016 Normally loaded by a prelude file, review where to load this
begin
require 'unicode_normalize'
rescue LoadError
end
4 changes: 4 additions & 0 deletions truffle/src/main/ruby/core/string.rb
Original file line number Diff line number Diff line change
@@ -847,11 +847,15 @@ def sub!(pattern, replacement=undefined)
if use_yield || hash
Regexp.last_match = match

duped = dup
if use_yield
val = yield match.to_s
else
val = hash[match.to_s]
end
if duped != self
raise RuntimeError, "string modified"
end
untrusted = true if val.untrusted?
val = val.to_s unless val.kind_of?(String)

0 comments on commit fbb6893

Please sign in to comment.