Skip to content
Permalink

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: jruby/jruby
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: 6155a494da09
Choose a base ref
...
head repository: jruby/jruby
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: a77c3fed567a
Choose a head ref
  • 5 commits
  • 3 files changed
  • 1 contributor

Commits on May 4, 2016

  1. Copy the full SHA
    54ca55e View commit details
  2. Copy the full SHA
    14883b6 View commit details
  3. Copy the full SHA
    0dc101a View commit details
  4. Copy the full SHA
    56acf8b View commit details
  5. Copy the full SHA
    a77c3fe View commit details
Original file line number Diff line number Diff line change
@@ -28,6 +28,7 @@
import org.jruby.truffle.core.numeric.BignumOperations;
import org.jruby.truffle.core.rope.CodeRange;
import org.jruby.truffle.core.rope.Rope;
import org.jruby.truffle.core.rope.RopeOperations;
import org.jruby.truffle.core.string.CoreStrings;
import org.jruby.truffle.core.string.StringOperations;
import org.jruby.truffle.extra.AttachmentsManager;
@@ -89,6 +90,10 @@ protected DynamicObject createString(ByteList bytes) {
return StringOperations.createString(getContext(), bytes);
}

protected DynamicObject createString(byte[] bytes, Encoding encoding) {
return StringOperations.createString(getContext(), RopeOperations.create(bytes, encoding, CodeRange.CR_7BIT));
}

protected DynamicObject create7BitString(CharSequence value, Encoding encoding) {
return StringOperations.createString(getContext(), StringOperations.encodeRope(value, encoding, CodeRange.CR_7BIT));
}
Original file line number Diff line number Diff line change
@@ -59,7 +59,6 @@
import org.jruby.truffle.core.cast.ToStrNode;
import org.jruby.truffle.core.cast.ToStrNodeGen;
import org.jruby.truffle.core.string.StringOperations;
import org.jruby.truffle.debug.DebugHelpers;
import org.jruby.truffle.language.NotProvided;
import org.jruby.truffle.language.RubyGuards;
import org.jruby.truffle.language.SnippetNode;
@@ -162,34 +161,26 @@ public Object parse(

final StreamReader reader;

// fall back on IOInputStream, using default charset
if (!RubyGuards.isRubyString(yaml) && respondToReadNode.doesRespondTo(frame, "read", yaml)) {
//final boolean isIO = (boolean) ruby("yaml.is_a? IO", "yaml", yaml);
//Encoding enc = isIO
// ? UTF8Encoding.INSTANCE // ((RubyIO)yaml).getReadEncoding()
// : UTF8Encoding.INSTANCE;
final Encoding enc = UTF8Encoding.INSTANCE;
Charset charset = enc.getCharset();
final Charset charset = enc.getCharset();
reader = new StreamReader(new InputStreamReader(new InputStreamAdapter(getContext(), yaml), charset));
} else {
ByteList byteList = StringOperations.getByteListReadOnly(toStrNode.coerceObject(frame, yaml));
Encoding enc = byteList.getEncoding();
Encoding encoding = byteList.getEncoding();

// if not unicode, transcode to UTF8
if (!(enc instanceof UnicodeEncoding)) {
byteList = EncodingUtils.strConvEnc(getContext().getJRubyRuntime().getCurrentContext(), byteList, enc, UTF8Encoding.INSTANCE);
enc = UTF8Encoding.INSTANCE;
}

ByteArrayInputStream bais = new ByteArrayInputStream(byteList.getUnsafeBytes(), byteList.getBegin(), byteList.getRealSize());

Charset charset = enc.getCharset();
if (!(encoding instanceof UnicodeEncoding)) {
byteList = EncodingUtils.strConvEnc(getContext().getJRubyRuntime().getCurrentContext(),
byteList, encoding, UTF8Encoding.INSTANCE);

assert charset != null : "charset for encoding " + enc + " should not be null";

InputStreamReader isr = new InputStreamReader(bais, charset);
encoding = UTF8Encoding.INSTANCE;
}

reader = new StreamReader(isr);
reader = new StreamReader(
new InputStreamReader(
new ByteArrayInputStream(
byteList.getUnsafeBytes(), byteList.getBegin(), byteList.getRealSize()),
encoding.getCharset()));
}

final Parser parser = new ParserImpl(reader);
@@ -204,58 +195,79 @@ public Object parse(
while (true) {
Event event = parser.getEvent();

// FIXME: Event should expose a getID, so it can be switched
if (event.is(Event.ID.StreamStart)) {
callStartStreamNode.call(frame, handler, "start_stream", null, YAMLEncoding.YAML_ANY_ENCODING.ordinal());
} else if (event.is(Event.ID.DocumentStart)) {
DumperOptions.Version _version = ((DocumentStartEvent) event).getVersion();
Integer[] versionInts = _version == null ? null : _version.getArray();
Object version = versionInts == null ?
Layouts.ARRAY.createArray(coreLibrary().getArrayFactory(), null, 0) :
Layouts.ARRAY.createArray(coreLibrary().getArrayFactory(), new Object[]{ versionInts[0], versionInts[1] }, 2);
final DocumentStartEvent startEvent = (DocumentStartEvent) event;

final DumperOptions.Version versionOptions = startEvent.getVersion();
final Integer[] versionInts = versionOptions == null ? null : versionOptions.getArray();

Map<String, String> tagsMap = ((DocumentStartEvent) event).getTags();
final DynamicObject versionArray;

if (versionInts == null) {
versionArray = Layouts.ARRAY.createArray(coreLibrary().getArrayFactory(), null, 0);
} else {
versionArray = Layouts.ARRAY.createArray(coreLibrary().getArrayFactory(), new Object[]{
versionInts[0], versionInts[1]
}, 2);
}

Map<String, String> tagsMap = startEvent.getTags();
DynamicObject tags = Layouts.ARRAY.createArray(coreLibrary().getArrayFactory(), null, 0);

if (tagsMap != null && tagsMap.size() > 0) {
for (Map.Entry<String, String> tag : tagsMap.entrySet()) {
Object key = stringFor(tag.getKey(), tainted, taintNode);
Object value = stringFor(tag.getValue(), tainted, taintNode);
tagPushNode.execute(frame, "tags.push [key, value]", "tags", tags, "key", key, "value", value);
tagPushNode.execute(frame,
"tags.push [key, value]",
"tags", tags,
"key", key,
"value", value);
}
}
Object notExplicit = !((DocumentStartEvent) event).getExplicit();

callStartDocumentNode.call(frame, handler, "start_document", null, version, tags, notExplicit);
Object notExplicit = !startEvent.getExplicit();
callStartDocumentNode.call(frame, handler, "start_document", null, versionArray, tags, notExplicit);
} else if (event.is(Event.ID.DocumentEnd)) {
Object notExplicit = !((DocumentEndEvent) event).getExplicit();
final DocumentEndEvent endEvent = (DocumentEndEvent) event;
Object notExplicit = !endEvent.getExplicit();
callEndDocumentNode.call(frame, handler, "end_document", null, notExplicit);
} else if (event.is(Event.ID.Alias)) {
Object alias = stringOrNilFor(((AliasEvent) event).getAnchor(), tainted, taintNode);
final AliasEvent aliasEvent = (AliasEvent) event;
Object alias = stringOrNilFor(aliasEvent.getAnchor(), tainted, taintNode);
callAliasNode.call(frame, handler, "alias", null, alias);
} else if (event.is(Event.ID.Scalar)) {
Object anchor = stringOrNilFor(((ScalarEvent) event).getAnchor(), tainted, taintNode);
Object tag = stringOrNilFor(((ScalarEvent) event).getTag(), tainted, taintNode);
Object plain_implicit = ((ScalarEvent) event).getImplicit().canOmitTagInPlainScalar();
Object quoted_implicit = ((ScalarEvent) event).getImplicit().canOmitTagInNonPlainScalar();
Object style = translateStyle(((ScalarEvent) event).getStyle());
Object val = stringFor(((ScalarEvent) event).getValue(), tainted, taintNode);
final ScalarEvent scalarEvent = (ScalarEvent) event;

Object anchor = stringOrNilFor(scalarEvent.getAnchor(), tainted, taintNode);
Object tag = stringOrNilFor(scalarEvent.getTag(), tainted, taintNode);
Object plain_implicit = scalarEvent.getImplicit().canOmitTagInPlainScalar();
Object quoted_implicit = scalarEvent.getImplicit().canOmitTagInNonPlainScalar();
Object style = translateStyle(scalarEvent.getStyle());
Object val = stringFor(scalarEvent.getValue(), tainted, taintNode);

callScalarNode.call(frame, handler, "scalar", null, val, anchor, tag, plain_implicit,
quoted_implicit, style);
} else if (event.is(Event.ID.SequenceStart)) {
Object anchor = stringOrNilFor(((SequenceStartEvent) event).getAnchor(), tainted, taintNode);
Object tag = stringOrNilFor(((SequenceStartEvent) event).getTag(), tainted, taintNode);
Object implicit = ((SequenceStartEvent) event).getImplicit();
Object style = translateFlowStyle(((SequenceStartEvent) event).getFlowStyle());
final SequenceStartEvent sequenceStartEvent = (SequenceStartEvent) event;

Object anchor = stringOrNilFor(sequenceStartEvent.getAnchor(), tainted, taintNode);
Object tag = stringOrNilFor(sequenceStartEvent.getTag(), tainted, taintNode);
Object implicit = sequenceStartEvent.getImplicit();
Object style = translateFlowStyle(sequenceStartEvent.getFlowStyle());

callStartSequenceNode.call(frame, handler, "start_sequence", null, anchor, tag, implicit, style);
} else if (event.is(Event.ID.SequenceEnd)) {
callEndSequenceNode.call(frame, handler, "end_sequence", null);
} else if (event.is(Event.ID.MappingStart)) {
Object anchor = stringOrNilFor(((MappingStartEvent) event).getAnchor(), tainted, taintNode);
Object tag = stringOrNilFor(((MappingStartEvent) event).getTag(), tainted, taintNode);
Object implicit = ((MappingStartEvent) event).getImplicit();
Object style = translateFlowStyle(((MappingStartEvent) event).getFlowStyle());
final MappingStartEvent mappingStartEvent = (MappingStartEvent) event;

Object anchor = stringOrNilFor(mappingStartEvent.getAnchor(), tainted, taintNode);
Object tag = stringOrNilFor(mappingStartEvent.getTag(), tainted, taintNode);
Object implicit = mappingStartEvent.getImplicit();
Object style = translateFlowStyle(mappingStartEvent.getFlowStyle());

callStartMappingNode.call(frame, handler, "start_mapping", null, anchor, tag, implicit, style);
} else if (event.is(Event.ID.MappingEnd)) {
@@ -268,20 +280,34 @@ public Object parse(
} catch (ParserException pe) {
final Mark mark = pe.getProblemMark();

Object[] arguments = new Object[]{"file", path, "line", mark.getLine(), "col", mark.getColumn(), "offset", mark.getIndex(), "problem", pe.getProblem() == null ? nil() : createString(new ByteList(pe.getProblem().getBytes(StandardCharsets.UTF_8))), "context", pe.getContext() == null ? nil() : createString(new ByteList(pe.getContext().getBytes(StandardCharsets.UTF_8)))};
raiseSyntaxErrorSnippetNode.execute(frame, "raise Psych::SyntaxError.new(file, line, col, offset, problem, context)", arguments);
raiseSyntaxErrorSnippetNode.execute(frame,
"raise Psych::SyntaxError.new(file, line, col, offset, problem, context)",
"file", path,
"line", mark.getLine(),
"col", mark.getColumn(),
"offset", mark.getIndex(),
"problem", pe.getProblem() == null ? nil() : createString(new ByteList(pe.getProblem().getBytes(StandardCharsets.UTF_8))),
"context", pe.getContext() == null ? nil() : createString(new ByteList(pe.getContext().getBytes(StandardCharsets.UTF_8))));
} catch (ScannerException se) {
StringBuilder message = new StringBuilder("syntax error");
if (se.getProblemMark() != null) {
message.append(se.getProblemMark().toString());
}
final Mark mark = se.getProblemMark();

Object[] arguments = new Object[]{"file", path, "line", mark.getLine(), "col", mark.getColumn(), "offset", mark.getIndex(), "problem", se.getProblem() == null ? nil() : createString(new ByteList(se.getProblem().getBytes(StandardCharsets.UTF_8))), "context", se.getContext() == null ? nil() : createString(new ByteList(se.getContext().getBytes(StandardCharsets.UTF_8)))};
raiseSyntaxErrorSnippetNode.execute(frame, "raise Psych::SyntaxError.new(file, line, col, offset, problem, context)", arguments);
raiseSyntaxErrorSnippetNode.execute(frame,
"raise Psych::SyntaxError.new(file, line, col, offset, problem, context)",
"file", path,
"line", mark.getLine(),
"col", mark.getColumn(),
"offset", mark.getIndex(),
"problem", se.getProblem() == null ? nil() : createString(new ByteList(se.getProblem().getBytes(StandardCharsets.UTF_8))),
"context", se.getContext() == null ? nil() : createString(new ByteList(se.getContext().getBytes(StandardCharsets.UTF_8))));
} catch (ReaderException re) {
Object[] arguments = new Object[]{"file", path, "line", 0, "col", 0, "offset", re.getPosition(), "problem", re.getName() == null ? nil() : createString(new ByteList(re.getName().getBytes(StandardCharsets.UTF_8))), "context", re.toString() == null ? nil() : createString(new ByteList(re.toString().getBytes(StandardCharsets.UTF_8)))};
raiseSyntaxErrorSnippetNode.execute(frame, "raise Psych::SyntaxError.new(file, line, col, offset, problem, context)", arguments);
raiseSyntaxErrorSnippetNode.execute(frame,
"raise Psych::SyntaxError.new(file, line, col, offset, problem, context)",
"file", path,
"line", 0,
"col", 0,
"offset", re.getPosition(),
"problem", re.getName() == null ? nil() : createString(new ByteList(re.getName().getBytes(StandardCharsets.UTF_8))),
"context", re.toString() == null ? nil() : createString(new ByteList(re.toString().getBytes(StandardCharsets.UTF_8))));
} catch (Throwable t) {
Helpers.throwException(t);
return parserObject;
@@ -298,53 +324,64 @@ protected TaintNode createTaintNode() {
return TaintNodeGen.create(getContext(), null, null);
}

private static int translateStyle(Character style) {
if (style == null) return 0; // any
private static final int STYLE_PLAIN = 1;
private static final int STYLE_SINGLE_QUOTED = 2;
private static final int STYLE_DOUBLE_QUOTED = 3;
private static final int STYLE_LITERAL = 4;
private static final int STYLE_FOLDED = 5;
private static final int STYLE_ANY = 0;
private static final int STYLE_FLOW = 2;
private static final int STYLE_NOT_FLOW = 1;

private static int translateStyle(Character style) {
switch (style) {
case 0:
return 1; // plain
return STYLE_PLAIN;
case '\'':
return 2; // single-quoted
return STYLE_SINGLE_QUOTED;
case '"':
return 3; // double-quoted
return STYLE_DOUBLE_QUOTED;
case '|':
return 4; // literal
return STYLE_LITERAL;
case '>':
return 5; // folded
return STYLE_FOLDED;
default:
return 0; // any
return STYLE_ANY;
}
}

private static int translateFlowStyle(Boolean flowStyle) {
if (flowStyle == null) return 0; // any

if (flowStyle) return 2;
return 1;
if (flowStyle == null) {
return STYLE_ANY;
} else if (flowStyle) {
return STYLE_FLOW;
} else {
return STYLE_NOT_FLOW;
}
}

private Object stringOrNilFor(String value, boolean tainted, TaintNode taintNode) {
if (value == null) return nil(); // No need to taint nil

return stringFor(value, tainted, taintNode);
if (value == null) {
return nil();
} else {
return stringFor(value, tainted, taintNode);
}
}

private Object stringFor(String value, boolean tainted, TaintNode taintNode) {
// TODO CS 23-Sep-15 this is JRuby's internal encoding, not ours
Encoding encoding = getContext().getJRubyRuntime().getDefaultInternalEncoding();

if (encoding == null) {
encoding = UTF8Encoding.INSTANCE;
}

Charset charset = RubyEncoding.UTF8;

if (encoding.getCharset() != null) {
charset = encoding.getCharset();
}

ByteList bytes = new ByteList(value.getBytes(charset), encoding);
Object string = createString(bytes);
final Object string = createString(value.getBytes(charset), encoding);

if (tainted) {
taintNode.executeTaint(string);
2 changes: 1 addition & 1 deletion truffle/src/main/ruby/core/truffle/cext.rb
Original file line number Diff line number Diff line change
@@ -112,7 +112,7 @@ def rb_define_module_under(mod, name)

def rb_define_method(mod, name, function, args)
mod.send(:define_method, name) do |*args|
Truffle::Interop.execute(function, self, *args)
function.call(self, *args)
end
end