Skip to content

Commit

Permalink
Showing 18 changed files with 494 additions and 353 deletions.
1 change: 1 addition & 0 deletions .travis.yml
Original file line number Diff line number Diff line change
@@ -9,6 +9,7 @@ cache:

before_install:
- unset GEM_PATH GEM_HOME IRBRC JRUBY_OPTS
- rm ~/.m2/settings.xml

jdk: oraclejdk8

4 changes: 2 additions & 2 deletions core/pom.rb
Original file line number Diff line number Diff line change
@@ -49,8 +49,8 @@
jar 'com.github.jnr:jnr-enxio:0.13', :exclusions => ['com.github.jnr:jnr-ffi']
jar 'com.github.jnr:jnr-x86asm:1.0.2', :exclusions => ['com.github.jnr:jnr-ffi']
jar 'com.github.jnr:jnr-unixsocket:0.14', :exclusions => ['com.github.jnr:jnr-ffi']
jar 'com.github.jnr:jnr-posix:3.0.31', :exclusions => ['com.github.jnr:jnr-ffi']
jar 'com.github.jnr:jnr-constants:0.9.4', :exclusions => ['com.github.jnr:jnr-ffi']
jar 'com.github.jnr:jnr-posix:3.0.32-SNAPSHOT', :exclusions => ['com.github.jnr:jnr-ffi']
jar 'com.github.jnr:jnr-constants:0.9.5-SNAPSHOT', :exclusions => ['com.github.jnr:jnr-ffi']
jar 'com.github.jnr:jnr-ffi:2.1.0'
jar 'com.github.jnr:jffi:${jffi.version}'
jar 'com.github.jnr:jffi:${jffi.version}:native'
4 changes: 2 additions & 2 deletions core/pom.xml
Original file line number Diff line number Diff line change
@@ -137,7 +137,7 @@ DO NOT MODIFIY - GENERATED CODE
<dependency>
<groupId>com.github.jnr</groupId>
<artifactId>jnr-posix</artifactId>
<version>3.0.31</version>
<version>3.0.32-SNAPSHOT</version>
<exclusions>
<exclusion>
<artifactId>jnr-ffi</artifactId>
@@ -148,7 +148,7 @@ DO NOT MODIFIY - GENERATED CODE
<dependency>
<groupId>com.github.jnr</groupId>
<artifactId>jnr-constants</artifactId>
<version>0.9.4</version>
<version>0.9.5-SNAPSHOT</version>
<exclusions>
<exclusion>
<artifactId>jnr-ffi</artifactId>
117 changes: 9 additions & 108 deletions core/src/main/java/org/jruby/runtime/encoding/EncodingService.java
Original file line number Diff line number Diff line change
@@ -3,7 +3,6 @@
import org.jcodings.Encoding;
import org.jcodings.EncodingDB;
import org.jcodings.EncodingDB.Entry;
import org.jcodings.ascii.AsciiTables;
import org.jcodings.specific.ASCIIEncoding;
import org.jcodings.specific.ISO8859_16Encoding;
import org.jcodings.util.CaseInsensitiveBytesHash;
@@ -19,22 +18,21 @@
import java.lang.reflect.Field;
import java.nio.charset.Charset;
import java.nio.charset.UnsupportedCharsetException;
import java.util.ArrayList;
import java.util.List;

import org.jcodings.specific.USASCIIEncoding;
import org.jruby.RubyFixnum;
import org.jruby.RubyString;
import org.jruby.ext.nkf.RubyNKF;
import org.jruby.util.SafePropertyAccessor;
import org.jruby.util.encoding.ISO_8859_16;
import org.jruby.util.io.EncodingUtils;

public final class EncodingService {
private final CaseInsensitiveBytesHash<Entry> encodings;
private final CaseInsensitiveBytesHash<Entry> aliases;

// for fast lookup: encoding entry => org.jruby.RubyEncoding
public final IRubyObject[] encodingList;
private final IRubyObject[] encodingList;
// for fast lookup: org.joni.encoding.Encoding => org.jruby.RubyEncoding
private RubyEncoding[] encodingIndex = new RubyEncoding[4];
// the runtime
@@ -178,133 +176,36 @@ public RubyEncoding getEncoding(Encoding enc) {
return encodingIndex[enc.getIndex()];
}

public interface EncodingDefinitionVisitor {
public void defineEncoding(Entry encodingEntry, byte[] name, int p, int end);

public void defineConstant(int encodingListIndex, String constName);
}

public interface EncodingAliasVisitor {
public void defineAlias(int encodingListIndex, String constName);

public void defineConstant(int encodingListIndex, String constName);
}

public void defineEncodings() {
defineEncodings(new EncodingDefinitionVisitor() {
@Override
public void defineEncoding(Entry encodingEntry, byte[] name, int p, int end) {
RubyEncoding encoding = RubyEncoding.newEncoding(runtime, name, p, end, encodingEntry.isDummy());
encodingList[encodingEntry.getIndex()] = encoding;
}

@Override
public void defineConstant(int encodingListIndex, String constName) {
defineEncodingConstant(runtime, (RubyEncoding) encodingList[encodingListIndex], constName);
}
});
}

public void defineEncodings(EncodingDefinitionVisitor visitor) {
HashEntryIterator hei = encodings.entryIterator();
while (hei.hasNext()) {
CaseInsensitiveBytesHash.CaseInsensitiveBytesHashEntry<Entry> e =
((CaseInsensitiveBytesHash.CaseInsensitiveBytesHashEntry<Entry>)hei.next());
Entry ee = e.value;

visitor.defineEncoding(ee, e.bytes, e.p, e.end);
RubyEncoding encoding = RubyEncoding.newEncoding(runtime, e.bytes, e.p, e.end, ee.isDummy());
encodingList[ee.getIndex()] = encoding;

for (String constName : encodingNames(e.bytes, e.p, e.end)) {
visitor.defineConstant(ee.getIndex(), constName);
for (String constName : EncodingUtils.encodingNames(e.bytes, e.p, e.end)) {
defineEncodingConstant(runtime, (RubyEncoding) encodingList[ee.getIndex()], constName);
}
}
}

public void defineAliases() {
defineAliases(new EncodingAliasVisitor() {
@Override
public void defineAlias(int encodingListIndex, String constName) { }

@Override
public void defineConstant(int encodingListIndex, String constName) {
defineEncodingConstant(runtime, (RubyEncoding) encodingList[encodingListIndex], constName);
}
});
}

public void defineAliases(EncodingAliasVisitor visitor) {
HashEntryIterator hei = aliases.entryIterator();
while (hei.hasNext()) {
CaseInsensitiveBytesHash.CaseInsensitiveBytesHashEntry<Entry> e =
((CaseInsensitiveBytesHash.CaseInsensitiveBytesHashEntry<Entry>)hei.next());
Entry ee = e.value;

for (String constName : encodingNames(e.bytes, e.p, e.end)) {
visitor.defineAlias(ee.getIndex(), constName);
visitor.defineConstant(ee.getIndex(), constName);
// The constant names must be treated by the the <code>encodingNames</code> helper.
for (String constName : EncodingUtils.encodingNames(e.bytes, e.p, e.end)) {
defineEncodingConstant(runtime, (RubyEncoding) encodingList[ee.getIndex()], constName);
}
}
}

private List<String> encodingNames(byte[] name, int p, int end) {
final List<String> names = new ArrayList<String>();

Encoding enc = ASCIIEncoding.INSTANCE;
int s = p;

int code = name[s] & 0xff;
if (enc.isDigit(code)) return names;

boolean hasUpper = false;
boolean hasLower = false;
if (enc.isUpper(code)) {
hasUpper = true;
while (++s < end && (enc.isAlnum(name[s] & 0xff) || name[s] == (byte)'_')) {
if (enc.isLower(name[s] & 0xff)) hasLower = true;
}
}

boolean isValid = false;
if (s >= end) {
isValid = true;
names.add(new String(name, p, end));
}

if (!isValid || hasLower) {
if (!hasLower || !hasUpper) {
do {
code = name[s] & 0xff;
if (enc.isLower(code)) hasLower = true;
if (enc.isUpper(code)) hasUpper = true;
} while (++s < end && (!hasLower || !hasUpper));
}

byte[]constName = new byte[end - p];
System.arraycopy(name, p, constName, 0, end - p);
s = 0;
code = constName[s] & 0xff;

if (!isValid) {
if (enc.isLower(code)) constName[s] = AsciiTables.ToUpperCaseTable[code];
for (; s < constName.length; ++s) {
if (!enc.isAlnum(constName[s] & 0xff)) constName[s] = (byte)'_';
}
if (hasUpper) {
names.add(new String(constName, 0, constName.length));
}
}
if (hasLower) {
for (s = 0; s < constName.length; ++s) {
code = constName[s] & 0xff;
if (enc.isLower(code)) constName[s] = AsciiTables.ToUpperCaseTable[code];
}
names.add(new String(constName, 0, constName.length));
}
}

return names;
}

private void defineEncodingConstant(Ruby runtime, RubyEncoding encoding, String constName) {
runtime.getEncoding().defineConstant(constName, encoding);
}
61 changes: 61 additions & 0 deletions core/src/main/java/org/jruby/util/io/EncodingUtils.java
Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@
import org.jcodings.Encoding;
import org.jcodings.EncodingDB;
import org.jcodings.Ptr;
import org.jcodings.ascii.AsciiTables;
import org.jcodings.specific.ASCIIEncoding;
import org.jcodings.specific.USASCIIEncoding;
import org.jcodings.specific.UTF16BEEncoding;
@@ -47,6 +48,7 @@
import org.jruby.util.StringSupport;
import org.jruby.util.TypeConverter;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

@@ -1116,6 +1118,65 @@ public static RubyString setStrBuf(Ruby runtime, final IRubyObject obj, final in
return str;
}

public static List<String> encodingNames(byte[] name, int p, int end) {
final List<String> names = new ArrayList<String>();

Encoding enc = ASCIIEncoding.INSTANCE;
int s = p;

int code = name[s] & 0xff;
if (enc.isDigit(code)) return names;

boolean hasUpper = false;
boolean hasLower = false;
if (enc.isUpper(code)) {
hasUpper = true;
while (++s < end && (enc.isAlnum(name[s] & 0xff) || name[s] == (byte)'_')) {
if (enc.isLower(name[s] & 0xff)) hasLower = true;
}
}

boolean isValid = false;
if (s >= end) {
isValid = true;
names.add(new String(name, p, end));
}

if (!isValid || hasLower) {
if (!hasLower || !hasUpper) {
do {
code = name[s] & 0xff;
if (enc.isLower(code)) hasLower = true;
if (enc.isUpper(code)) hasUpper = true;
} while (++s < end && (!hasLower || !hasUpper));
}

byte[]constName = new byte[end - p];
System.arraycopy(name, p, constName, 0, end - p);
s = 0;
code = constName[s] & 0xff;

if (!isValid) {
if (enc.isLower(code)) constName[s] = AsciiTables.ToUpperCaseTable[code];
for (; s < constName.length; ++s) {
if (!enc.isAlnum(constName[s] & 0xff)) constName[s] = (byte)'_';
}
if (hasUpper) {
names.add(new String(constName, 0, constName.length));
}
}
if (hasLower) {
for (s = 0; s < constName.length; ++s) {
code = constName[s] & 0xff;
if (enc.isLower(code)) constName[s] = AsciiTables.ToUpperCaseTable[code];
}
names.add(new String(constName, 0, constName.length));
}
}

return names;
}

public interface ResizeFunction {
/**
* Resize the destination, returning the new begin offset.
4 changes: 4 additions & 0 deletions truffle/src/main/java/org/jruby/truffle/RubyContext.java
Original file line number Diff line number Diff line change
@@ -150,6 +150,10 @@ public RubyContext(Ruby jrubyRuntime, TruffleLanguage.Env env) {
nativePlatform = NativePlatformFactory.createPlatform(this);
rootLexicalScope = new LexicalScope(null, coreLibrary.getObjectClass());

// The encoding manager relies on POSIX having been initialized, so we can't process it during
// normal core library initialization.
coreLibrary.initializeEncodingManager();

threadManager = new ThreadManager(this);
threadManager.initialize();

86 changes: 66 additions & 20 deletions truffle/src/main/java/org/jruby/truffle/core/CoreLibrary.java
Original file line number Diff line number Diff line change
@@ -27,11 +27,12 @@
import org.jcodings.EncodingDB;
import org.jcodings.specific.UTF8Encoding;
import org.jcodings.transcode.EConvFlags;
import org.jcodings.util.CaseInsensitiveBytesHash;
import org.jcodings.util.Hash;
import org.jruby.Main;
import org.jruby.ext.ffi.Platform;
import org.jruby.ext.ffi.Platform.OS_TYPE;
import org.jruby.runtime.Constants;
import org.jruby.runtime.encoding.EncodingService;
import org.jruby.truffle.Layouts;
import org.jruby.truffle.RubyContext;
import org.jruby.truffle.RubyLanguage;
@@ -47,7 +48,10 @@
import org.jruby.truffle.core.bool.TrueClassNodesFactory;
import org.jruby.truffle.core.dir.DirNodesFactory;
import org.jruby.truffle.core.encoding.EncodingConverterNodesFactory;
import org.jruby.truffle.core.encoding.EncodingManager;
import org.jruby.truffle.core.encoding.EncodingNodesFactory;
import org.jruby.truffle.core.encoding.EncodingOperations;
import org.jruby.truffle.core.encoding.TruffleEncodingNodesFactory;
import org.jruby.truffle.core.exception.ExceptionNodesFactory;
import org.jruby.truffle.core.exception.NameErrorNodesFactory;
import org.jruby.truffle.core.exception.NoMethodErrorNodesFactory;
@@ -135,6 +139,7 @@
import org.jruby.truffle.stdlib.psych.PsychParserNodesFactory;
import org.jruby.truffle.stdlib.psych.YAMLEncoding;
import org.jruby.util.cli.OutputStrings;
import org.jruby.util.io.EncodingUtils;

import java.io.File;
import java.io.IOException;
@@ -586,6 +591,7 @@ public CoreLibrary(RubyContext context) {
defineModule(truffleModule, "Digest");
defineModule(truffleModule, "ObjSpace");
defineModule(truffleModule, "Etc");
defineModule(truffleModule, "Encoding");
defineModule(truffleModule, "Coverage");
defineModule(truffleModule, "Graal");
defineModule(truffleModule, "Ropes");
@@ -690,7 +696,6 @@ private void includeModules(DynamicObject comparableModule) {
public void initialize() {
initializeGlobalVariables();
initializeConstants();
initializeEncodingConstants();
initializeSignalConstants();
}

@@ -765,6 +770,7 @@ public void addCoreMethods(PrimitiveManager primitiveManager) {
TruffleBindingNodesFactory.getFactories(),
TruffleBootNodesFactory.getFactories(),
TruffleDebugNodesFactory.getFactories(),
TruffleEncodingNodesFactory.getFactories(),
TruffleFixnumNodesFactory.getFactories(),
TruffleGCNodesFactory.getFactories(),
TruffleGraalNodesFactory.getFactories(),
@@ -1078,32 +1084,72 @@ public void initializePostBoot() {
}
}

public void initializeEncodingConstants() {
getContext().getJRubyRuntime().getEncodingService().defineEncodings(new EncodingService.EncodingDefinitionVisitor() {
@Override
public void defineEncoding(EncodingDB.Entry encodingEntry, byte[] name, int p, int end) {
context.getEncodingManager().defineEncoding(encodingEntry, name, p, end);
private void initializeEncodings() {
final EncodingManager encodingManager = context.getEncodingManager();
final Hash.HashEntryIterator hei = EncodingDB.getEncodings().entryIterator();

while (hei.hasNext()) {
final CaseInsensitiveBytesHash.CaseInsensitiveBytesHashEntry<EncodingDB.Entry> e =
((CaseInsensitiveBytesHash.CaseInsensitiveBytesHashEntry<EncodingDB.Entry>)hei.next());
final EncodingDB.Entry encodingEntry = e.value;

encodingManager.defineEncoding(encodingEntry, e.bytes, e.p, e.end);

for (String constName : EncodingUtils.encodingNames(e.bytes, e.p, e.end)) {
final DynamicObject rubyEncoding = context.getEncodingManager().getRubyEncoding(encodingEntry.getIndex());
Layouts.MODULE.getFields(encodingClass).setConstant(context, node, constName, rubyEncoding);
}
}
}

private void initializeEncodingAliases() {
final EncodingManager encodingManager = context.getEncodingManager();
final Hash.HashEntryIterator hei = EncodingDB.getAliases().entryIterator();

while (hei.hasNext()) {
final CaseInsensitiveBytesHash.CaseInsensitiveBytesHashEntry<EncodingDB.Entry> e =
((CaseInsensitiveBytesHash.CaseInsensitiveBytesHashEntry<EncodingDB.Entry>)hei.next());
final EncodingDB.Entry encodingEntry = e.value;

// The alias name should be exactly the one in the encodings DB.
encodingManager.defineAlias(encodingEntry.getIndex(), new String(e.bytes, e.p, e.end));

@Override
public void defineConstant(int encodingListIndex, String constName) {
final DynamicObject rubyEncoding = context.getEncodingManager().getRubyEncoding(encodingListIndex);
// The constant names must be treated by the the <code>encodingNames</code> helper.
for (String constName : EncodingUtils.encodingNames(e.bytes, e.p, e.end)) {
final DynamicObject rubyEncoding = context.getEncodingManager().getRubyEncoding(encodingEntry.getIndex());
Layouts.MODULE.getFields(encodingClass).setConstant(context, node, constName, rubyEncoding);
}
});
}
}

public void initializeEncodingManager() {
initializeEncodings();
initializeEncodingAliases();

getContext().getJRubyRuntime().getEncodingService().defineAliases(new EncodingService.EncodingAliasVisitor() {
@Override
public void defineAlias(int encodingListIndex, String constName) {
context.getEncodingManager().defineAlias(encodingListIndex, constName);
// External should always have a value, but Encoding.external_encoding{,=} will lazily setup
final String externalEncodingName = getContext().getJRubyRuntime().getInstanceConfig().getExternalEncoding();
if (externalEncodingName != null && !externalEncodingName.equals("")) {
final DynamicObject loadedEncoding = getContext().getEncodingManager().getRubyEncoding(externalEncodingName);
if (loadedEncoding == null) {
// TODO (nirvdrum 28-Oct-16): This should just print a nice error message and exit with a status code of 1 -- it's essentially an input validation error -- no need to show the user a full trace.
throw new RuntimeException("unknown encoding name - " + externalEncodingName);
} else {
getContext().getEncodingManager().setDefaultExternalEncoding(EncodingOperations.getEncoding(loadedEncoding));
}
} else {
getContext().getEncodingManager().setDefaultExternalEncoding(getContext().getEncodingManager().getLocaleEncoding());
}

@Override
public void defineConstant(int encodingListIndex, String constName) {
final DynamicObject rubyEncoding = context.getEncodingManager().getRubyEncoding(encodingListIndex);
Layouts.MODULE.getFields(encodingClass).setConstant(context, node, constName, rubyEncoding);
final String internalEncodingName = getContext().getJRubyRuntime().getInstanceConfig().getInternalEncoding();
if (internalEncodingName != null && !internalEncodingName.equals("")) {
final DynamicObject rubyEncoding = getContext().getEncodingManager().getRubyEncoding(internalEncodingName);
if (rubyEncoding == null) {
// TODO (nirvdrum 28-Oct-16): This should just print a nice error message and exit with a status code of 1 -- it's essentially an input validation error -- no need to show the user a full trace.
throw new RuntimeException("unknown encoding name - " + internalEncodingName);
} else {
getContext().getEncodingManager().setDefaultInternalEncoding(EncodingOperations.getEncoding(rubyEncoding));
}
});
}
}

@TruffleBoundary
Original file line number Diff line number Diff line change
@@ -17,7 +17,6 @@
import com.oracle.truffle.api.object.DynamicObject;
import com.oracle.truffle.api.source.SourceSection;
import org.jcodings.Encoding;
import org.jcodings.EncodingDB;
import org.jcodings.Ptr;
import org.jcodings.specific.USASCIIEncoding;
import org.jcodings.transcode.EConv;
@@ -26,9 +25,7 @@
import org.jcodings.transcode.TranscoderDB;
import org.jcodings.util.CaseInsensitiveBytesHash;
import org.jcodings.util.Hash;
import org.jruby.Ruby;
import org.jruby.runtime.Visibility;
import org.jruby.runtime.builtin.IRubyObject;
import org.jruby.truffle.Layouts;
import org.jruby.truffle.RubyContext;
import org.jruby.truffle.builtins.CoreClass;
@@ -51,56 +48,34 @@
import org.jruby.truffle.language.objects.AllocateObjectNode;
import org.jruby.truffle.util.StringUtils;
import org.jruby.util.ByteList;
import org.jruby.util.io.EncodingUtils;

import static org.jruby.truffle.core.string.StringOperations.rope;

@CoreClass("Encoding::Converter")
public abstract class EncodingConverterNodes {

@NonStandard
@CoreMethod(names = "initialize_jruby", required = 2, optional = 1, visibility = Visibility.PRIVATE)
@CoreMethod(names = "initialize_jruby", required = 2, optional = 1, lowerFixnum = 3, visibility = Visibility.PRIVATE)
public abstract static class InitializeNode extends CoreMethodArrayArgumentsNode {

@TruffleBoundary
@Specialization
public DynamicObject initialize(DynamicObject self, Object source, Object destination, Object unusedOptions) {
@Specialization(guards = { "isRubyEncoding(source)", "isRubyEncoding(destination)" })
public DynamicObject initialize(DynamicObject self, DynamicObject source, DynamicObject destination, int options) {
// Adapted from RubyConverter - see attribution there

Ruby runtime = getContext().getJRubyRuntime();
Encoding[] encs = {null, null};
byte[][] encNames = {null, null};
int[] ecflags = {0};
IRubyObject[] ecopts = {runtime.getNil()};

final IRubyObject sourceAsJRubyObj = toJRuby(source);
final IRubyObject destinationAsJRubyObj = toJRuby(destination);

EncodingUtils.econvArgs(runtime.getCurrentContext(), new IRubyObject[]{sourceAsJRubyObj, destinationAsJRubyObj}, encNames, encs, ecflags, ecopts);

//
// This method should only be called after the Encoding::Converter instance has already been initialized
// by Rubinius. Rubinius will do the heavy lifting of parsing the options hash and setting the `@options`
// ivar to the resulting int for EConv flags. Since we don't pass the proper data structures to EncodingUtils,
// we must override the flags after its had a pass in order to correct the bad flags value.
ecflags[0] = rubiniusToJRubyFlags((int) self.get("@options", coreLibrary().getNilObject()));
// ivar to the resulting int for EConv flags.

EConv econv = EncodingUtils.econvOpenOpts(runtime.getCurrentContext(), encNames[0], encNames[1], ecflags[0], ecopts[0]);
Encoding sourceEncoding = Layouts.ENCODING.getEncoding(source);
Encoding destinationEncoding = Layouts.ENCODING.getEncoding(destination);
final byte[] sourceEncodingName = sourceEncoding.getName();
final byte[] destinationEncodingName = destinationEncoding.getName();

if (econv == null) {
throw new UnsupportedOperationException();
}
final EConv econv = TranscoderDB.open(sourceEncodingName, destinationEncodingName, rubiniusToJRubyFlags(options));

if (!EncodingUtils.DECORATOR_P(encNames[0], encNames[1])) {
if (encs[0] == null) {
encs[0] = EncodingDB.dummy(encNames[0]).getEncoding();
}
if (encs[1] == null) {
encs[1] = EncodingDB.dummy(encNames[1]).getEncoding();
}
}

econv.sourceEncoding = encs[0];
econv.destinationEncoding = encs[1];
econv.sourceEncoding = sourceEncoding;
econv.destinationEncoding = destinationEncoding;

Layouts.ENCODING_CONVERTER.setEconv(self, econv);

@@ -124,17 +99,6 @@ private int rubiniusToJRubyFlags(int flags) {
return flags;
}

private IRubyObject toJRuby(Object object) {
if (RubyGuards.isRubyString(object)) {
return getContext().getJRubyRuntime().newString(RopeOperations.toByteListCopy(StringOperations.rope((DynamicObject) object)));
} else if (RubyGuards.isRubyEncoding(object)) {
final Rope rope = StringOperations.rope(Layouts.ENCODING.getName((DynamicObject) object));
return getContext().getJRubyRuntime().getEncodingService().rubyEncodingFromObject(getContext().getJRubyRuntime().newString(RopeOperations.toByteListCopy(rope)));
} else {
throw new UnsupportedOperationException();
}
}

}

@NonStandard
Original file line number Diff line number Diff line change
@@ -14,6 +14,7 @@

import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
import com.oracle.truffle.api.object.DynamicObject;
import jnr.constants.platform.LangInfo;
import org.jcodings.Encoding;
import org.jcodings.EncodingDB;
import org.jcodings.EncodingDB.Entry;
@@ -44,6 +45,9 @@ public class EncodingManager {

private final RubyContext context;

private Encoding defaultExternalEncoding;
private Encoding defaultInternalEncoding;

public EncodingManager(RubyContext context) {
this.context = context;
}
@@ -114,7 +118,20 @@ public synchronized DynamicObject replicateEncoding(Encoding encoding, String na

@TruffleBoundary
public Encoding getLocaleEncoding() {
return context.getJRubyRuntime().getEncodingService().getLocaleEncoding();
String localeEncodingName;
try {
localeEncodingName = context.getNativePlatform().getPosix().nl_langinfo(LangInfo.CODESET.intValue());
}
catch (UnsupportedOperationException e) {
localeEncodingName = Charset.defaultCharset().name();
}

DynamicObject rubyEncoding = getRubyEncoding(localeEncodingName);
if (rubyEncoding == null) {
rubyEncoding = getRubyEncoding("US-ASCII");
}

return EncodingOperations.getEncoding(rubyEncoding);
}

@TruffleBoundary
@@ -136,4 +153,19 @@ public static Charset charsetForEncoding(Encoding encoding) {
}
}

public void setDefaultExternalEncoding(Encoding defaultExternalEncoding) {
this.defaultExternalEncoding = defaultExternalEncoding;
}

public Encoding getDefaultExternalEncoding() {
return defaultExternalEncoding;
}

public void setDefaultInternalEncoding(Encoding defaultInternalEncoding) {
this.defaultInternalEncoding = defaultInternalEncoding;
}

public Encoding getDefaultInternalEncoding() {
return defaultInternalEncoding;
}
}
Original file line number Diff line number Diff line change
@@ -11,8 +11,6 @@
*/
package org.jruby.truffle.core.encoding;

import com.oracle.truffle.api.CompilerAsserts;
import com.oracle.truffle.api.CompilerDirectives;
import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
import com.oracle.truffle.api.dsl.Cached;
import com.oracle.truffle.api.dsl.NodeChild;
@@ -26,30 +24,22 @@
import org.jcodings.EncodingDB;
import org.jcodings.EncodingDB.Entry;
import org.jcodings.specific.USASCIIEncoding;
import org.jcodings.util.CaseInsensitiveBytesHash.CaseInsensitiveBytesHashEntry;
import org.jcodings.util.Hash.HashEntry;
import org.jruby.runtime.Visibility;
import org.jruby.truffle.Layouts;
import org.jruby.truffle.RubyContext;
import org.jruby.truffle.builtins.CoreClass;
import org.jruby.truffle.builtins.CoreMethod;
import org.jruby.truffle.builtins.CoreMethodArrayArgumentsNode;
import org.jruby.truffle.builtins.NonStandard;
import org.jruby.truffle.builtins.Primitive;
import org.jruby.truffle.builtins.PrimitiveArrayArgumentsNode;
import org.jruby.truffle.builtins.UnaryCoreMethodNode;
import org.jruby.truffle.builtins.YieldingCoreMethodNode;
import org.jruby.truffle.core.cast.ToEncodingNode;
import org.jruby.truffle.core.cast.ToStrNode;
import org.jruby.truffle.core.cast.ToStrNodeGen;
import org.jruby.truffle.core.rope.CodeRange;
import org.jruby.truffle.core.rope.Rope;
import org.jruby.truffle.core.string.StringOperations;
import org.jruby.truffle.language.RubyGuards;
import org.jruby.truffle.language.RubyNode;
import org.jruby.truffle.language.SnippetNode;
import org.jruby.truffle.language.control.RaiseException;
import org.jruby.util.ByteList;

@CoreClass("Encoding")
public abstract class EncodingNodes {
@@ -326,84 +316,6 @@ protected int getCacheLimit() {

}

@NonStandard
@CoreMethod(names = "default_external_jruby=", onSingleton = true, required = 1)
public abstract static class SetDefaultExternalNode extends CoreMethodArrayArgumentsNode {

@Child private ToStrNode toStrNode;

@TruffleBoundary
@Specialization(guards = "isRubyEncoding(encoding)")
public DynamicObject defaultExternalEncoding(DynamicObject encoding) {
getContext().getJRubyRuntime().setDefaultExternalEncoding(EncodingOperations.getEncoding(encoding));

return encoding;
}

@TruffleBoundary
@Specialization(guards = "isRubyString(encodingString)")
public DynamicObject defaultExternal(DynamicObject encodingString) {
final DynamicObject rubyEncoding = getContext().getEncodingManager().getRubyEncoding(encodingString.toString());
getContext().getJRubyRuntime().setDefaultExternalEncoding(EncodingOperations.getEncoding(rubyEncoding));

return rubyEncoding;
}

@TruffleBoundary
@Specialization(guards = "isNil(nil)")
public DynamicObject defaultExternal(Object nil) {
throw new RaiseException(coreExceptions().argumentError("default external can not be nil", this));
}

@Specialization(guards = { "!isRubyEncoding(encoding)", "!isRubyString(encoding)", "!isNil(encoding)" })
public DynamicObject defaultExternal(VirtualFrame frame, Object encoding) {
if (toStrNode == null) {
CompilerDirectives.transferToInterpreterAndInvalidate();
toStrNode = insert(ToStrNodeGen.create(getContext(), null, null));
}

return defaultExternal(toStrNode.executeToStr(frame, encoding));
}

}

@NonStandard
@CoreMethod(names = "default_internal_jruby=", onSingleton = true, required = 1)
public abstract static class SetDefaultInternalNode extends CoreMethodArrayArgumentsNode {

@Child private ToStrNode toStrNode;

@TruffleBoundary
@Specialization(guards = "isRubyEncoding(encoding)")
public DynamicObject defaultInternal(DynamicObject encoding) {
getContext().getJRubyRuntime().setDefaultInternalEncoding(EncodingOperations.getEncoding(encoding));

return encoding;
}

@TruffleBoundary
@Specialization(guards = "isNil(encoding)")
public DynamicObject defaultInternal(Object encoding) {
getContext().getJRubyRuntime().setDefaultInternalEncoding(null);

return nil();
}

@Specialization(guards = { "!isRubyEncoding(encoding)", "!isNil(encoding)" })
public DynamicObject defaultInternal(VirtualFrame frame, Object encoding) {
if (toStrNode == null) {
CompilerDirectives.transferToInterpreterAndInvalidate();
toStrNode = insert(ToStrNodeGen.create(getContext(), null, null));
}

final DynamicObject encodingName = toStrNode.executeToStr(frame, encoding);
getContext().getJRubyRuntime().setDefaultInternalEncoding(EncodingOperations.getEncoding(getContext().getEncodingManager().getRubyEncoding(encodingName.toString())));

return encodingName;
}

}

@CoreMethod(names = "list", onSingleton = true)
public abstract static class ListNode extends CoreMethodArrayArgumentsNode {

@@ -452,52 +364,6 @@ protected static boolean isDummy(DynamicObject encoding) {
}
}

@NonStandard
@CoreMethod(names = "each_alias", onSingleton = true, visibility = Visibility.PRIVATE, needsBlock = true)
public abstract static class EachAliasNode extends YieldingCoreMethodNode {

@Specialization
public DynamicObject eachAlias(VirtualFrame frame, DynamicObject block) {
CompilerAsserts.neverPartOfCompilation();
for (HashEntry<Entry> entry : EncodingDB.getAliases().entryIterator()) {
final CaseInsensitiveBytesHashEntry<Entry> e = (CaseInsensitiveBytesHashEntry<Entry>) entry;
final ByteList aliasName = new ByteList(e.bytes, e.p, e.end - e.p, USASCIIEncoding.INSTANCE, false);
yield(frame, block, createString(aliasName), entry.value.getIndex());
}
return nil();
}
}

@NonStandard
@CoreMethod(names = "get_default_encoding", onSingleton = true, visibility = Visibility.PRIVATE, required = 1)
public abstract static class GetDefaultEncodingNode extends CoreMethodArrayArgumentsNode {

@Specialization(guards = "isRubyString(name)")
public DynamicObject getDefaultEncoding(DynamicObject name) {
final Encoding encoding = getEncoding(StringOperations.getString(name));
if (encoding == null) {
return nil();
} else {
return getContext().getEncodingManager().getRubyEncoding(encoding);
}
}

@TruffleBoundary
private Encoding getEncoding(String name) {
switch (name) {
case "internal":
return getContext().getJRubyRuntime().getDefaultInternalEncoding();
case "external":
return getContext().getJRubyRuntime().getDefaultExternalEncoding();
case "locale":
case "filesystem":
return getContext().getEncodingManager().getLocaleEncoding();
default:
throw new UnsupportedOperationException();
}
}
}

@CoreMethod(names = { "name", "to_s" })
public abstract static class ToSNode extends CoreMethodArrayArgumentsNode {

Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
/*
* Copyright (c) 2014, 2016 Oracle and/or its affiliates. All rights reserved. This
* code is released under a tri EPL/GPL/LGPL license. You can use it,
* redistribute it and/or modify it under the terms of the:
*
* Eclipse Public License version 1.0
* GNU General Public License version 2
* GNU Lesser General Public License version 2.1
*/
package org.jruby.truffle.core.encoding;

import com.oracle.truffle.api.CompilerAsserts;
import com.oracle.truffle.api.CompilerDirectives;
import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
import com.oracle.truffle.api.dsl.Specialization;
import com.oracle.truffle.api.frame.VirtualFrame;
import com.oracle.truffle.api.object.DynamicObject;
import org.jcodings.Encoding;
import org.jcodings.EncodingDB;
import org.jcodings.specific.USASCIIEncoding;
import org.jcodings.util.CaseInsensitiveBytesHash;
import org.jcodings.util.Hash;
import org.jruby.truffle.builtins.CoreClass;
import org.jruby.truffle.builtins.CoreMethod;
import org.jruby.truffle.builtins.CoreMethodArrayArgumentsNode;
import org.jruby.truffle.builtins.YieldingCoreMethodNode;
import org.jruby.truffle.core.cast.ToStrNode;
import org.jruby.truffle.core.cast.ToStrNodeGen;
import org.jruby.truffle.core.string.StringOperations;
import org.jruby.truffle.language.control.RaiseException;
import org.jruby.util.ByteList;

@CoreClass("Truffle::Encoding")
public abstract class TruffleEncodingNodes {

@CoreMethod(names = "default_external=", onSingleton = true, required = 1)
public abstract static class SetDefaultExternalNode extends CoreMethodArrayArgumentsNode {

@Child private ToStrNode toStrNode;

@Specialization(guards = "isRubyEncoding(encoding)")
public DynamicObject defaultExternalEncoding(DynamicObject encoding) {
getContext().getEncodingManager().setDefaultExternalEncoding(EncodingOperations.getEncoding(encoding));

return encoding;
}

@Specialization(guards = "isRubyString(encodingString)")
public DynamicObject defaultExternal(DynamicObject encodingString) {
final DynamicObject rubyEncoding = getContext().getEncodingManager().getRubyEncoding(StringOperations.getString(encodingString));
getContext().getEncodingManager().setDefaultExternalEncoding(EncodingOperations.getEncoding(rubyEncoding));

return rubyEncoding;
}

@Specialization(guards = "isNil(nil)")
public DynamicObject defaultExternal(Object nil) {
throw new RaiseException(coreExceptions().argumentError("default external can not be nil", this));
}

@Specialization(guards = { "!isRubyEncoding(encoding)", "!isRubyString(encoding)", "!isNil(encoding)" })
public DynamicObject defaultExternal(VirtualFrame frame, Object encoding) {
if (toStrNode == null) {
CompilerDirectives.transferToInterpreterAndInvalidate();
toStrNode = insert(ToStrNodeGen.create(getContext(), null, null));
}

return defaultExternal(toStrNode.executeToStr(frame, encoding));
}

}

@CoreMethod(names = "default_internal=", onSingleton = true, required = 1)
public abstract static class SetDefaultInternalNode extends CoreMethodArrayArgumentsNode {

@Child private ToStrNode toStrNode;

@Specialization(guards = "isRubyEncoding(encoding)")
public DynamicObject defaultInternal(DynamicObject encoding) {
getContext().getEncodingManager().setDefaultInternalEncoding(EncodingOperations.getEncoding(encoding));

return encoding;
}

@Specialization(guards = "isNil(encoding)")
public DynamicObject defaultInternal(Object encoding) {
getContext().getEncodingManager().setDefaultInternalEncoding(null);

return nil();
}

@Specialization(guards = { "!isRubyEncoding(encoding)", "!isNil(encoding)" })
public DynamicObject defaultInternal(VirtualFrame frame, Object encoding) {
if (toStrNode == null) {
CompilerDirectives.transferToInterpreterAndInvalidate();
toStrNode = insert(ToStrNodeGen.create(getContext(), null, null));
}

final DynamicObject encodingName = toStrNode.executeToStr(frame, encoding);
getContext().getJRubyRuntime().setDefaultInternalEncoding(EncodingOperations.getEncoding(getContext().getEncodingManager().getRubyEncoding(encodingName.toString())));

return encodingName;
}

}

@CoreMethod(names = "each_alias", onSingleton = true, needsBlock = true)
public abstract static class EachAliasNode extends YieldingCoreMethodNode {

@Specialization
public DynamicObject eachAlias(VirtualFrame frame, DynamicObject block) {
CompilerAsserts.neverPartOfCompilation();
for (Hash.HashEntry<EncodingDB.Entry> entry : EncodingDB.getAliases().entryIterator()) {
final CaseInsensitiveBytesHash.CaseInsensitiveBytesHashEntry<EncodingDB.Entry> e = (CaseInsensitiveBytesHash.CaseInsensitiveBytesHashEntry<EncodingDB.Entry>) entry;
final ByteList aliasName = new ByteList(e.bytes, e.p, e.end - e.p, USASCIIEncoding.INSTANCE, false);
yield(frame, block, createString(aliasName), entry.value.getIndex());
}
return nil();
}
}

@CoreMethod(names = "get_default_encoding", onSingleton = true, required = 1)
public abstract static class GetDefaultEncodingNode extends CoreMethodArrayArgumentsNode {

@Specialization(guards = "isRubyString(name)")
public DynamicObject getDefaultEncoding(DynamicObject name) {
final Encoding encoding = getEncoding(StringOperations.getString(name));
if (encoding == null) {
return nil();
} else {
return getContext().getEncodingManager().getRubyEncoding(encoding);
}
}

@TruffleBoundary
private Encoding getEncoding(String name) {
switch (name) {
case "internal":
return getContext().getEncodingManager().getDefaultInternalEncoding();
case "external":
return getContext().getEncodingManager().getDefaultExternalEncoding();
case "locale":
case "filesystem":
return getContext().getEncodingManager().getLocaleEncoding();
default:
throw new UnsupportedOperationException();
}
}
}


}
Original file line number Diff line number Diff line change
@@ -856,7 +856,7 @@ public DynamicObject gets() {
// TODO(CS): having some trouble interacting with JRuby stdin - so using this hack
final InputStream in = getContext().getJRubyRuntime().getInstanceConfig().getInput();

Encoding encoding = getContext().getJRubyRuntime().getDefaultExternalEncoding();
Encoding encoding = getContext().getEncodingManager().getDefaultExternalEncoding();

final BufferedReader reader = new BufferedReader(new InputStreamReader(in, encoding.getCharset()));

Original file line number Diff line number Diff line change
@@ -1176,25 +1176,6 @@ private Object substr(Rope rope, DynamicObject string, int beg, int len) {
}
}

@CoreMethod(names = "inspect")
public abstract static class InspectNode extends CoreMethodArrayArgumentsNode {

@Child private TaintResultNode taintResultNode;

public InspectNode(RubyContext context, SourceSection sourceSection) {
super(context, sourceSection);
taintResultNode = new TaintResultNode(context, sourceSection);
}

@TruffleBoundary
@Specialization
public Object inspect(DynamicObject string) {
DynamicObject result = createString(org.jruby.RubyString.inspect(getContext().getJRubyRuntime(),
StringOperations.getByteListReadOnly(string)).getByteList());
return taintResultNode.maybeTaint(string, result);
}
}

@CoreMethod(names = "encoding")
public abstract static class EncodingNode extends CoreMethodArrayArgumentsNode {

Original file line number Diff line number Diff line change
@@ -439,4 +439,9 @@ public int mkfifo(String path, int mode) {
public long[] getgroups() {
return posix.getgroups();
}

@Override
public String nl_langinfo(int item) {
return posix.nl_langinfo(item);
}
}
Original file line number Diff line number Diff line change
@@ -104,5 +104,6 @@ public interface TrufflePosix {
int isatty(int fd);
int mkfifo(String path, int mode);
long[] getgroups();
String nl_langinfo(int item);

}
Original file line number Diff line number Diff line change
@@ -410,7 +410,7 @@ private Object stringOrNilFor(String value, boolean tainted, TaintNode taintNode

@TruffleBoundary
private Object stringFor(String value, boolean tainted, TaintNode taintNode) {
Encoding encoding = getContext().getJRubyRuntime().getDefaultInternalEncoding();
Encoding encoding = getContext().getEncodingManager().getDefaultInternalEncoding();

if (encoding == null) {
encoding = UTF8Encoding.INSTANCE;
21 changes: 4 additions & 17 deletions truffle/src/main/ruby/core/encoding.rb
Original file line number Diff line number Diff line change
@@ -62,14 +62,14 @@ def build_encoding_map
map[key] = [nil, index]
}

each_alias { |alias_name, index|
Truffle::Encoding.each_alias { |alias_name, index|
key = alias_name.upcase.to_sym
map[key] = [alias_name, index]
}

%w[internal external locale filesystem].each { |name|
key = name.upcase.to_sym
enc = get_default_encoding(name)
enc = Truffle::Encoding.get_default_encoding(name)
index = enc ? map[enc.name.upcase.to_sym].last : nil
map[key] = [name, index]
}
@@ -588,6 +588,7 @@ def self.default_external=(enc)
set_alias_index "external", enc
set_alias_index "filesystem", enc
@default_external = undefined
Truffle::Encoding.default_external = enc
end

def self.default_internal
@@ -600,6 +601,7 @@ def self.default_internal
def self.default_internal=(enc)
set_alias_index "internal", enc
@default_internal = undefined
Truffle::Encoding.default_internal = enc
end

def self.find(name)
@@ -641,22 +643,7 @@ def _dump(depth)
def self._load(name)
find name
end

class << self
alias_method :default_external_rubinius=, :default_external=

def default_external=(enc)
self.default_external_rubinius = enc
self.default_external_jruby = enc
end

alias_method :default_internal_rubinius=, :default_internal=

def default_internal=(enc)
self.default_internal_rubinius = enc
self.default_internal_jruby = enc
end
end
end

Encoding::TranscodingMap[:'UTF-16BE'] = Rubinius::LookupTable.new
140 changes: 140 additions & 0 deletions truffle/src/main/ruby/core/string.rb
Original file line number Diff line number Diff line change
@@ -569,6 +569,146 @@ def end_with?(*suffixes)
false
end

def inspect
result_encoding = Encoding.default_internal || Encoding.default_external
unless result_encoding.ascii_compatible?
result_encoding = Encoding::US_ASCII
end

enc = encoding
ascii = enc.ascii_compatible?
enc_name = enc.name
unicode = enc_name.start_with?("UTF-") && enc_name[4] != ?7

if unicode
if enc.equal? Encoding::UTF_16
a = getbyte 0
b = getbyte 1

if a == 0xfe and b == 0xff
enc = Encoding::UTF_16BE
elsif a == 0xff and b == 0xfe
enc = Encoding::UTF_16LE
else
unicode = false
end
elsif enc.equal? Encoding::UTF_32
a = getbyte 0
b = getbyte 1
c = getbyte 2
d = getbyte 3

if a == 0 and b == 0 and c == 0xfe and d == 0xfe
enc = Encoding::UTF_32BE
elsif a == 0xff and b == 0xfe and c == 0 and d == 0
enc = Encoding::UTF_32LE
else
unicode = false
end
end
end

array = []

index = 0
total = bytesize
while index < total
char = chr_at index

if char
bs = char.bytesize

if (ascii or unicode) and bs == 1
escaped = nil

byte = getbyte(index)
if byte >= 7 and byte <= 92
case byte
when 7 # \a
escaped = '\a'
when 8 # \b
escaped = '\b'
when 9 # \t
escaped = '\t'
when 10 # \n
escaped = '\n'
when 11 # \v
escaped = '\v'
when 12 # \f
escaped = '\f'
when 13 # \r
escaped = '\r'
when 27 # \e
escaped = '\e'
when 34 # \"
escaped = '\"'
when 35 # #
case getbyte(index += 1)
when 36 # $
escaped = '\#$'
when 64 # @
escaped = '\#@'
when 123 # {
escaped = '\#{'
else
index -= 1
end
when 92 # \\
escaped = '\\\\'
end

if escaped
array << escaped
index += 1
next
end
end
end

if char.printable?
array << char
else
code = char.ord
escaped = code.to_s(16).upcase

if unicode
if code < 0x10000
pad = "0" * (4 - escaped.bytesize)
array << "\\u#{pad}#{escaped}"
else
array << "\\u{#{escaped}}"
end
else
if code < 0x100
pad = "0" * (2 - escaped.bytesize)
array << "\\x#{pad}#{escaped}"
else
array << "\\x{#{escaped}}"
end
end
end

index += bs
else
array << "\\x#{getbyte(index).to_s(16)}"
index += 1
end
end

size = array.inject(0) { |s, chr| s += chr.bytesize }
result = String.pattern size + 2, ?".ord
m = Rubinius::Mirror.reflect result

index = 1
array.each do |chr|
m.copy_from chr, 0, chr.bytesize, index
index += chr.bytesize
end

Rubinius::Type.infect result, self
result.force_encoding result_encoding
end

def prepend(other)
self[0, 0] = other
self

0 comments on commit 2dcbc66

Please sign in to comment.