-
-
Notifications
You must be signed in to change notification settings - Fork 925
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
First prototype of flatbuffer-based persistence.
This prototype obviously isn't including everything needed to fully interpret or reconstitute the full scope, but it does express structure. FlatIRWriter contains a main that persists a simple script and then accesses that persisted IR directly from a memory-mapped buffer. If the back-and-forth through the flatbuffer does not introduce too much overhead, this should mean we can start to explore interpreting directly from disk, without any pass to physically construct all IR objects right away. We will be able to start interpreting immediately upon opening the persisted file, and then lazily stand up the richer structures needed for full interp and JIT.
Showing
12 changed files
with
414 additions
and
13 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
112 changes: 112 additions & 0 deletions
112
core/src/main/java/org/jruby/ir/persistence/flat/FlatIRWriter.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,112 @@ | ||
| package org.jruby.ir.persistence.flat; | ||
|
|
||
| import com.google.flatbuffers.FlatBufferBuilder; | ||
| import org.jruby.ParseResult; | ||
| import org.jruby.Ruby; | ||
| import org.jruby.ast.RootNode; | ||
| import org.jruby.ir.IRBuilder; | ||
| import org.jruby.ir.IRScope; | ||
| import org.jruby.ir.IRScriptBody; | ||
| import org.jruby.ir.Operation; | ||
| import org.jruby.ir.instructions.Instr; | ||
| import org.jruby.ir.interpreter.InterpreterContext; | ||
| import org.jruby.ir.operands.Operand; | ||
| import org.jruby.ir.operands.OperandType; | ||
|
|
||
| import java.io.ByteArrayInputStream; | ||
| import java.io.FileInputStream; | ||
| import java.io.FileOutputStream; | ||
| import java.io.RandomAccessFile; | ||
| import java.nio.ByteBuffer; | ||
| import java.nio.channels.FileChannel; | ||
|
|
||
| public class FlatIRWriter { | ||
| public static int createIRScopeFlat(FlatBufferBuilder builder, IRScope scope) { | ||
| int nameOffset = builder.createString(scope.getName()); | ||
|
|
||
| int[] children = new int[scope.getLexicalScopes().size()]; | ||
|
|
||
| for (int i = 0; i < scope.getLexicalScopes().size(); i++) { | ||
| IRScope child = scope.getLexicalScopes().get(i); | ||
|
|
||
| children[i] = createIRScopeFlat(builder, child); | ||
| } | ||
|
|
||
| int childrenOffset = IRScopeFlat.createLexicalChildrenVector(builder, children); | ||
|
|
||
| int closuresOffset = IRScopeFlat.createNestedClosuresVector(builder, new int[0]); | ||
|
|
||
| Instr[] instrs = scope.getInterpreterContext().getInstructions(); | ||
| int[] instrOffsets = new int[instrs.length]; | ||
|
|
||
| for (int i = 0; i < instrs.length; i++) { | ||
| Instr instr = instrs[i]; | ||
|
|
||
| if (instr.getOperation().getFlat() == -1) throw new RuntimeException("unsupported operation: " + instr.getOperation()); | ||
|
|
||
| Operand[] operands = instr.getOperands(); | ||
| int[] operandOffsets = new int[operands.length]; | ||
|
|
||
| for (int j = 0; j < operands.length; j++) { | ||
| Operand operand = operands[j]; | ||
|
|
||
| if (operand.getOperandType().getFlat() == -1) throw new RuntimeException("unsupported operand: " + operand.getOperandType()); | ||
|
|
||
| operandOffsets[j] = OperandFlat.createOperandFlat(builder, operand.getOperandType().getFlat()); | ||
| } | ||
|
|
||
| int operandsOffset = InstrFlat.createOperandsVector(builder, operandOffsets); | ||
|
|
||
| instrOffsets[i] = InstrFlat.createInstrFlat(builder, instr.getOperation().getFlat(), operandsOffset); | ||
| } | ||
|
|
||
| int instrsOffset = IRScopeFlat.createInstrsVector(builder, instrOffsets); | ||
|
|
||
| return IRScopeFlat.createIRScopeFlat(builder, nameOffset, closuresOffset, childrenOffset, instrsOffset); | ||
| } | ||
|
|
||
| public static void main(String[] args) { | ||
| Ruby runtime = Ruby.newInstance(); | ||
| byte[] src = "puts 'hello'".getBytes(); | ||
| ParseResult result = runtime.parseFromMain("blah.rb", new ByteArrayInputStream(src)); | ||
| InterpreterContext ic = IRBuilder.buildRoot(runtime.getIRManager(), (RootNode) result); | ||
| IRScope scope = ic.getScope(); | ||
| FlatBufferBuilder builder = new FlatBufferBuilder(); | ||
| int index = createIRScopeFlat(builder, scope); | ||
|
|
||
| builder.finish(index); | ||
|
|
||
| ByteBuffer buffer = builder.dataBuffer(); | ||
| int size = buffer.limit(); | ||
|
|
||
| try { | ||
| FileOutputStream out = new FileOutputStream("blah.ir"); | ||
| out.getChannel().write(buffer); | ||
| out.close(); | ||
| } catch (Throwable t) { | ||
| throw new RuntimeException(t); | ||
| } | ||
|
|
||
| try { | ||
| RandomAccessFile file = new RandomAccessFile("blah.ir", "rw"); | ||
| buffer = file.getChannel().map(FileChannel.MapMode.READ_ONLY, 0, size); | ||
| } catch (Throwable t) { | ||
| throw new RuntimeException(t); | ||
| } | ||
|
|
||
| System.out.println("persisted and loaded:"); | ||
| IRScopeFlat scopeFlat = IRScopeFlat.getRootAsIRScopeFlat(buffer); | ||
| System.out.println("IRScope: " + scopeFlat.name()); | ||
| System.out.println("Instructions:"); | ||
| InstrFlat instrFlat = new InstrFlat(); | ||
| for (int i = 0; i < scopeFlat.instrsLength(); i++) { | ||
| scopeFlat.instrs(instrFlat, i); | ||
| System.out.println(" " + Operation.flatMap(instrFlat.operation())); | ||
| OperandFlat operandFlat = new OperandFlat(); | ||
| for (int j = 0; j < instrFlat.operandsLength(); j++) { | ||
| instrFlat.operands(operandFlat, j); | ||
| System.out.println(" " + OperandType.flatMap(operandFlat.operandType())); | ||
| } | ||
| } | ||
| } | ||
| } |
33 changes: 33 additions & 0 deletions
33
core/src/main/java/org/jruby/ir/persistence/flat/IRClosureFlat.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| // automatically generated, do not modify | ||
|
|
||
| package org.jruby.ir.persistence.flat; | ||
|
|
||
| import java.nio.*; | ||
| import java.lang.*; | ||
| import java.util.*; | ||
| import com.google.flatbuffers.*; | ||
|
|
||
| @SuppressWarnings("unused") | ||
| public final class IRClosureFlat extends Table { | ||
| public static IRClosureFlat getRootAsIRClosureFlat(ByteBuffer _bb) { return getRootAsIRClosureFlat(_bb, new IRClosureFlat()); } | ||
| public static IRClosureFlat getRootAsIRClosureFlat(ByteBuffer _bb, IRClosureFlat obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__init(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } | ||
| public IRClosureFlat __init(int _i, ByteBuffer _bb) { bb_pos = _i; bb = _bb; return this; } | ||
|
|
||
| public IRScopeFlat scope() { return scope(new IRScopeFlat()); } | ||
| public IRScopeFlat scope(IRScopeFlat obj) { int o = __offset(4); return o != 0 ? obj.__init(__indirect(o + bb_pos), bb) : null; } | ||
|
|
||
| public static int createIRClosureFlat(FlatBufferBuilder builder, | ||
| int scopeOffset) { | ||
| builder.startObject(1); | ||
| IRClosureFlat.addScope(builder, scopeOffset); | ||
| return IRClosureFlat.endIRClosureFlat(builder); | ||
| } | ||
|
|
||
| public static void startIRClosureFlat(FlatBufferBuilder builder) { builder.startObject(1); } | ||
| public static void addScope(FlatBufferBuilder builder, int scopeOffset) { builder.addOffset(0, scopeOffset, 0); } | ||
| public static int endIRClosureFlat(FlatBufferBuilder builder) { | ||
| int o = builder.endObject(); | ||
| return o; | ||
| } | ||
| }; | ||
|
|
58 changes: 58 additions & 0 deletions
58
core/src/main/java/org/jruby/ir/persistence/flat/IRScopeFlat.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| // automatically generated, do not modify | ||
|
|
||
| package org.jruby.ir.persistence.flat; | ||
|
|
||
| import java.nio.*; | ||
| import java.lang.*; | ||
| import java.util.*; | ||
| import com.google.flatbuffers.*; | ||
|
|
||
| @SuppressWarnings("unused") | ||
| public final class IRScopeFlat extends Table { | ||
| public static IRScopeFlat getRootAsIRScopeFlat(ByteBuffer _bb) { return getRootAsIRScopeFlat(_bb, new IRScopeFlat()); } | ||
| public static IRScopeFlat getRootAsIRScopeFlat(ByteBuffer _bb, IRScopeFlat obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__init(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } | ||
| public IRScopeFlat __init(int _i, ByteBuffer _bb) { bb_pos = _i; bb = _bb; return this; } | ||
|
|
||
| public String name() { int o = __offset(4); return o != 0 ? __string(o + bb_pos) : null; } | ||
| public ByteBuffer nameAsByteBuffer() { return __vector_as_bytebuffer(4, 1); } | ||
| public IRClosureFlat nestedClosures(int j) { return nestedClosures(new IRClosureFlat(), j); } | ||
| public IRClosureFlat nestedClosures(IRClosureFlat obj, int j) { int o = __offset(6); return o != 0 ? obj.__init(__indirect(__vector(o) + j * 4), bb) : null; } | ||
| public int nestedClosuresLength() { int o = __offset(6); return o != 0 ? __vector_len(o) : 0; } | ||
| public IRScopeFlat lexicalChildren(int j) { return lexicalChildren(new IRScopeFlat(), j); } | ||
| public IRScopeFlat lexicalChildren(IRScopeFlat obj, int j) { int o = __offset(8); return o != 0 ? obj.__init(__indirect(__vector(o) + j * 4), bb) : null; } | ||
| public int lexicalChildrenLength() { int o = __offset(8); return o != 0 ? __vector_len(o) : 0; } | ||
| public InstrFlat instrs(int j) { return instrs(new InstrFlat(), j); } | ||
| public InstrFlat instrs(InstrFlat obj, int j) { int o = __offset(10); return o != 0 ? obj.__init(__indirect(__vector(o) + j * 4), bb) : null; } | ||
| public int instrsLength() { int o = __offset(10); return o != 0 ? __vector_len(o) : 0; } | ||
|
|
||
| public static int createIRScopeFlat(FlatBufferBuilder builder, | ||
| int nameOffset, | ||
| int nestedClosuresOffset, | ||
| int lexicalChildrenOffset, | ||
| int instrsOffset) { | ||
| builder.startObject(4); | ||
| IRScopeFlat.addInstrs(builder, instrsOffset); | ||
| IRScopeFlat.addLexicalChildren(builder, lexicalChildrenOffset); | ||
| IRScopeFlat.addNestedClosures(builder, nestedClosuresOffset); | ||
| IRScopeFlat.addName(builder, nameOffset); | ||
| return IRScopeFlat.endIRScopeFlat(builder); | ||
| } | ||
|
|
||
| public static void startIRScopeFlat(FlatBufferBuilder builder) { builder.startObject(4); } | ||
| public static void addName(FlatBufferBuilder builder, int nameOffset) { builder.addOffset(0, nameOffset, 0); } | ||
| public static void addNestedClosures(FlatBufferBuilder builder, int nestedClosuresOffset) { builder.addOffset(1, nestedClosuresOffset, 0); } | ||
| public static int createNestedClosuresVector(FlatBufferBuilder builder, int[] data) { builder.startVector(4, data.length, 4); for (int i = data.length - 1; i >= 0; i--) builder.addOffset(data[i]); return builder.endVector(); } | ||
| public static void startNestedClosuresVector(FlatBufferBuilder builder, int numElems) { builder.startVector(4, numElems, 4); } | ||
| public static void addLexicalChildren(FlatBufferBuilder builder, int lexicalChildrenOffset) { builder.addOffset(2, lexicalChildrenOffset, 0); } | ||
| public static int createLexicalChildrenVector(FlatBufferBuilder builder, int[] data) { builder.startVector(4, data.length, 4); for (int i = data.length - 1; i >= 0; i--) builder.addOffset(data[i]); return builder.endVector(); } | ||
| public static void startLexicalChildrenVector(FlatBufferBuilder builder, int numElems) { builder.startVector(4, numElems, 4); } | ||
| public static void addInstrs(FlatBufferBuilder builder, int instrsOffset) { builder.addOffset(3, instrsOffset, 0); } | ||
| public static int createInstrsVector(FlatBufferBuilder builder, int[] data) { builder.startVector(4, data.length, 4); for (int i = data.length - 1; i >= 0; i--) builder.addOffset(data[i]); return builder.endVector(); } | ||
| public static void startInstrsVector(FlatBufferBuilder builder, int numElems) { builder.startVector(4, numElems, 4); } | ||
| public static int endIRScopeFlat(FlatBufferBuilder builder) { | ||
| int o = builder.endObject(); | ||
| return o; | ||
| } | ||
| public static void finishIRScopeFlatBuffer(FlatBufferBuilder builder, int offset) { builder.finish(offset); } | ||
| }; | ||
|
|
40 changes: 40 additions & 0 deletions
40
core/src/main/java/org/jruby/ir/persistence/flat/InstrFlat.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| // automatically generated, do not modify | ||
|
|
||
| package org.jruby.ir.persistence.flat; | ||
|
|
||
| import java.nio.*; | ||
| import java.lang.*; | ||
| import java.util.*; | ||
| import com.google.flatbuffers.*; | ||
|
|
||
| @SuppressWarnings("unused") | ||
| public final class InstrFlat extends Table { | ||
| public static InstrFlat getRootAsInstrFlat(ByteBuffer _bb) { return getRootAsInstrFlat(_bb, new InstrFlat()); } | ||
| public static InstrFlat getRootAsInstrFlat(ByteBuffer _bb, InstrFlat obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__init(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } | ||
| public InstrFlat __init(int _i, ByteBuffer _bb) { bb_pos = _i; bb = _bb; return this; } | ||
|
|
||
| public short operation() { int o = __offset(4); return o != 0 ? bb.getShort(o + bb_pos) : 0; } | ||
| public OperandFlat operands(int j) { return operands(new OperandFlat(), j); } | ||
| public OperandFlat operands(OperandFlat obj, int j) { int o = __offset(6); return o != 0 ? obj.__init(__indirect(__vector(o) + j * 4), bb) : null; } | ||
| public int operandsLength() { int o = __offset(6); return o != 0 ? __vector_len(o) : 0; } | ||
|
|
||
| public static int createInstrFlat(FlatBufferBuilder builder, | ||
| short operation, | ||
| int operandsOffset) { | ||
| builder.startObject(2); | ||
| InstrFlat.addOperands(builder, operandsOffset); | ||
| InstrFlat.addOperation(builder, operation); | ||
| return InstrFlat.endInstrFlat(builder); | ||
| } | ||
|
|
||
| public static void startInstrFlat(FlatBufferBuilder builder) { builder.startObject(2); } | ||
| public static void addOperation(FlatBufferBuilder builder, short operation) { builder.addShort(0, operation, 0); } | ||
| public static void addOperands(FlatBufferBuilder builder, int operandsOffset) { builder.addOffset(1, operandsOffset, 0); } | ||
| public static int createOperandsVector(FlatBufferBuilder builder, int[] data) { builder.startVector(4, data.length, 4); for (int i = data.length - 1; i >= 0; i--) builder.addOffset(data[i]); return builder.endVector(); } | ||
| public static void startOperandsVector(FlatBufferBuilder builder, int numElems) { builder.startVector(4, numElems, 4); } | ||
| public static int endInstrFlat(FlatBufferBuilder builder) { | ||
| int o = builder.endObject(); | ||
| return o; | ||
| } | ||
| }; | ||
|
|
32 changes: 32 additions & 0 deletions
32
core/src/main/java/org/jruby/ir/persistence/flat/OperandFlat.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,32 @@ | ||
| // automatically generated, do not modify | ||
|
|
||
| package org.jruby.ir.persistence.flat; | ||
|
|
||
| import java.nio.*; | ||
| import java.lang.*; | ||
| import java.util.*; | ||
| import com.google.flatbuffers.*; | ||
|
|
||
| @SuppressWarnings("unused") | ||
| public final class OperandFlat extends Table { | ||
| public static OperandFlat getRootAsOperandFlat(ByteBuffer _bb) { return getRootAsOperandFlat(_bb, new OperandFlat()); } | ||
| public static OperandFlat getRootAsOperandFlat(ByteBuffer _bb, OperandFlat obj) { _bb.order(ByteOrder.LITTLE_ENDIAN); return (obj.__init(_bb.getInt(_bb.position()) + _bb.position(), _bb)); } | ||
| public OperandFlat __init(int _i, ByteBuffer _bb) { bb_pos = _i; bb = _bb; return this; } | ||
|
|
||
| public byte operandType() { int o = __offset(4); return o != 0 ? bb.get(o + bb_pos) : 0; } | ||
|
|
||
| public static int createOperandFlat(FlatBufferBuilder builder, | ||
| byte operandType) { | ||
| builder.startObject(1); | ||
| OperandFlat.addOperandType(builder, operandType); | ||
| return OperandFlat.endOperandFlat(builder); | ||
| } | ||
|
|
||
| public static void startOperandFlat(FlatBufferBuilder builder) { builder.startObject(1); } | ||
| public static void addOperandType(FlatBufferBuilder builder, byte operandType) { builder.addByte(0, operandType, 0); } | ||
| public static int endOperandFlat(FlatBufferBuilder builder) { | ||
| int o = builder.endObject(); | ||
| return o; | ||
| } | ||
| }; | ||
|
|
17 changes: 17 additions & 0 deletions
17
core/src/main/java/org/jruby/ir/persistence/flat/OperandTypeFlat.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| // automatically generated, do not modify | ||
|
|
||
| package org.jruby.ir.persistence.flat; | ||
|
|
||
| public final class OperandTypeFlat { | ||
| private OperandTypeFlat() { } | ||
| public static final byte TEMPORARY_VARIABLE = 0; | ||
| public static final byte CURRENT_SCOPE = 1; | ||
| public static final byte SCOPE_MODULE = 2; | ||
| public static final byte STRING_LITERAL = 3; | ||
| public static final byte SELF = 4; | ||
|
|
||
| private static final String[] names = { "TEMPORARY_VARIABLE", "CURRENT_SCOPE", "SCOPE_MODULE", "STRING_LITERAL", "SELF", }; | ||
|
|
||
| public static String name(int e) { return names[e]; } | ||
| }; | ||
|
|
19 changes: 19 additions & 0 deletions
19
core/src/main/java/org/jruby/ir/persistence/flat/OperationFlat.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| // automatically generated, do not modify | ||
|
|
||
| package org.jruby.ir.persistence.flat; | ||
|
|
||
| public final class OperationFlat { | ||
| private OperationFlat() { } | ||
| public static final short COPY = 0; | ||
| public static final short RECEIVE_SELF = 1; | ||
| public static final short LINE_NUMBER = 2; | ||
| public static final short CALL_1_OBJ = 3; | ||
| public static final short RETURN = 4; | ||
| public static final short LOAD_IMPLICIT_CLOSURE = 5; | ||
| public static final short LOAD_FRAME_CLOSURE = 6; | ||
|
|
||
| private static final String[] names = { "COPY", "RECEIVE_SELF", "LINE_NUMBER", "CALL_1_OBJ", "RETURN", "LOAD_IMPLICIT_CLOSURE", "LOAD_FRAME_CLOSURE", }; | ||
|
|
||
| public static String name(int e) { return names[e]; } | ||
| }; | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,32 @@ | ||
| namespace org.jruby.ir.persistence.flat; | ||
|
|
||
| table IRScopeFlat { | ||
| name:string; | ||
| //lexicalParent:IRScopeFlat; | ||
| nestedClosures:[IRClosureFlat]; | ||
| lexicalChildren:[IRScopeFlat]; | ||
| instrs:[InstrFlat]; | ||
| } | ||
|
|
||
| table IRClosureFlat { | ||
| scope:IRScopeFlat; | ||
| } | ||
|
|
||
| enum OperationFlat : short { | ||
| COPY, RECEIVE_SELF, LINE_NUMBER, CALL_1_OBJ, RETURN, LOAD_IMPLICIT_CLOSURE, LOAD_FRAME_CLOSURE | ||
| } | ||
|
|
||
| table InstrFlat { | ||
| operation:OperationFlat; | ||
| operands:[OperandFlat]; | ||
| } | ||
|
|
||
| enum OperandTypeFlat : byte { | ||
| TEMPORARY_VARIABLE, CURRENT_SCOPE, SCOPE_MODULE, STRING_LITERAL, SELF | ||
| } | ||
|
|
||
| table OperandFlat { | ||
| operandType:OperandTypeFlat; | ||
| } | ||
|
|
||
| root_type IRScopeFlat; |