Skip to content

Commit

Permalink
YAML revamp
Browse files Browse the repository at this point in the history 
- Add support for YAML core schema
- Add support for possibly other schemas via the abstract YAML::Parser class
- Add missing arguments to YAML::Builder (tag, anchor, scalar)
- Add support for parsing and generating recursive data structures
- Add support for merge (<<)
- Move LibYAML enums to the YAML namespace
@asterite
asterite authored and Martin Verzilli committed Oct 2, 2017
1 parent ab30165 commit 3335450
Showing 27 changed files with 2,508 additions and 489 deletions.
74 changes: 72 additions & 2 deletions spec/std/yaml/any_spec.cr
View file
Original file line number Diff line number Diff line change
@@ -24,6 +24,67 @@ describe YAML::Any do
YAML.parse("foo: bar").as_h?.should eq({"foo" => "bar"})
YAML.parse("foo: bar")["foo"].as_h?.should be_nil
end

it "gets int64" do
value = YAML.parse("1").as_i64
value.should eq(1)
value.should be_a(Int64)

value = YAML.parse("1").as_i64?
value.should eq(1)
value.should be_a(Int64)

value = YAML.parse("true").as_i64?
value.should be_nil
end

it "gets int32" do
value = YAML.parse("1").as_i
value.should eq(1)
value.should be_a(Int32)

value = YAML.parse("1").as_i?
value.should eq(1)
value.should be_a(Int32)

value = YAML.parse("true").as_i?
value.should be_nil
end

it "gets float64" do
value = YAML.parse("1.2").as_f
value.should eq(1.2)
value.should be_a(Float64)

value = YAML.parse("1.2").as_f?
value.should eq(1.2)
value.should be_a(Float64)

value = YAML.parse("true").as_f?
value.should be_nil
end

it "gets time" do
value = YAML.parse("2010-01-02").as_time
value.should eq(Time.new(2010, 1, 2, kind: Time::Kind::Utc))

value = YAML.parse("2010-01-02").as_time?
value.should eq(Time.new(2010, 1, 2, kind: Time::Kind::Utc))

value = YAML.parse("hello").as_time?
value.should be_nil
end

it "gets bytes" do
value = YAML.parse("!!binary aGVsbG8=").as_bytes
value.should eq("hello".to_slice)

value = YAML.parse("!!binary aGVsbG8=").as_bytes?
value.should eq("hello".to_slice)

value = YAML.parse("1").as_bytes?
value.should be_nil
end
end

describe "#size" do
@@ -44,6 +105,10 @@ describe YAML::Any do
it "of hash" do
YAML.parse("foo: bar")["foo"].raw.should eq("bar")
end

it "of hash with integer keys" do
YAML.parse("1: bar")[1].raw.should eq("bar")
end
end

describe "#[]?" do
@@ -56,6 +121,11 @@ describe YAML::Any do
YAML.parse("foo: bar")["foo"]?.not_nil!.raw.should eq("bar")
YAML.parse("foo: bar")["fox"]?.should be_nil
end

it "of hash with integer keys" do
YAML.parse("1: bar")[1]?.not_nil!.raw.should eq("bar")
YAML.parse("1: bar")[2]?.should be_nil
end
end

describe "each" do
@@ -94,7 +164,7 @@ describe YAML::Any do
end

it "can compare with ===" do
("1" === YAML.parse("1")).should be_truthy
(1 === YAML.parse("1")).should be_truthy
end

it "exposes $~ when doing Regex#===" do
@@ -106,7 +176,7 @@ describe YAML::Any do
nums = YAML.parse("[1, 2, 3]")
nums.each_with_index do |x, i|
x.should be_a(YAML::Any)
x.raw.should eq((i + 1).to_s)
x.raw.should eq(i + 1)
end
end
end
81 changes: 81 additions & 0 deletions spec/std/yaml/builder_spec.cr
View file
Original file line number Diff line number Diff line change
@@ -15,6 +15,24 @@ describe YAML::Builder do
end
end

it "writes scalar with style" do
assert_built(%(--- "1"\n)) do
scalar(1, style: YAML::ScalarStyle::DOUBLE_QUOTED)
end
end

it "writes scalar with tag" do
assert_built(%(--- !foo 1\n...\n)) do
scalar(1, tag: "!foo")
end
end

it "writes scalar with anchor" do
assert_built(%(--- &foo 1\n...\n)) do
scalar(1, anchor: "foo")
end
end

it "writes sequence" do
assert_built("---\n- 1\n- 2\n- 3\n") do
sequence do
@@ -25,6 +43,36 @@ describe YAML::Builder do
end
end

it "writes sequence with tag" do
assert_built("--- !foo\n- 1\n- 2\n- 3\n") do
sequence(tag: "!foo") do
scalar(1)
scalar(2)
scalar(3)
end
end
end

it "writes sequence with anchor" do
assert_built("--- &foo\n- 1\n- 2\n- 3\n") do
sequence(anchor: "foo") do
scalar(1)
scalar(2)
scalar(3)
end
end
end

it "writes sequence with style" do
assert_built("--- [1, 2, 3]\n") do
sequence(style: YAML::SequenceStyle::FLOW) do
scalar(1)
scalar(2)
scalar(3)
end
end
end

it "writes mapping" do
assert_built("---\nfoo: 1\nbar: 2\n") do
mapping do
@@ -35,4 +83,37 @@ describe YAML::Builder do
end
end
end

it "writes mapping with tag" do
assert_built("--- !foo\nfoo: 1\nbar: 2\n") do
mapping(tag: "!foo") do
scalar("foo")
scalar(1)
scalar("bar")
scalar(2)
end
end
end

it "writes mapping with anchor" do
assert_built("--- &foo\nfoo: 1\nbar: 2\n") do
mapping(anchor: "foo") do
scalar("foo")
scalar(1)
scalar("bar")
scalar(2)
end
end
end

it "writes mapping with style" do
assert_built("--- {foo: 1, bar: 2}\n") do
mapping(style: YAML::MappingStyle::FLOW) do
scalar("foo")
scalar(1)
scalar("bar")
scalar(2)
end
end
end
end
138 changes: 134 additions & 4 deletions spec/std/yaml/mapping_spec.cr
View file
Original file line number Diff line number Diff line change
@@ -96,6 +96,34 @@ private class YAMLWithPresence
})
end

class YAMLRecursive
YAML.mapping({
name: String,
other: YAMLRecursive,
})
end

class YAMLRecursiveNilable
YAML.mapping({
name: String,
other: YAMLRecursiveNilable?,
})
end

class YAMLRecursiveArray
YAML.mapping({
name: String,
other: Array(YAMLRecursiveArray),
})
end

class YAMLRecursiveHash
YAML.mapping({
name: String,
other: Hash(String, YAMLRecursiveHash),
})
end

describe "YAML mapping" do
it "parses person" do
person = YAMLPerson.from_yaml("---\nname: John\nage: 30\n")
@@ -127,6 +155,35 @@ describe "YAML mapping" do
people[1].name.should eq("Doe")
end

it "parses array of people with merge" do
yaml = <<-YAML
- &1
name: foo
age: 1
-
<<: *1
age: 2
YAML

people = Array(YAMLPerson).from_yaml(yaml)
people[1].name.should eq("foo")
people[1].age.should eq(2)
end

it "parses array of people with merge, doesn't hang on infinite recursion" do
yaml = <<-YAML
- &1
name: foo
<<: *1
<<: [ *1, *1 ]
age: 1
YAML

people = Array(YAMLPerson).from_yaml(yaml)
people[0].name.should eq("foo")
people[0].age.should eq(1)
end

it "parses person with unknown attributes" do
person = YAMLPerson.from_yaml("---\nname: John\nunknown: [1, 2, 3]\nage: 30\n")
person.should be_a(YAMLPerson)
@@ -197,6 +254,68 @@ describe "YAML mapping" do
typeof(yaml.bar).should eq(Int8)
end

it "parses recursive" do
yaml = <<-YAML
--- &1
name: foo
other: *1
YAML

rec = YAMLRecursive.from_yaml(yaml)
rec.name.should eq("foo")
rec.other.should be(rec)
end

it "parses recursive nilable (1)" do
yaml = <<-YAML
--- &1
name: foo
other: *1
YAML

rec = YAMLRecursiveNilable.from_yaml(yaml)
rec.name.should eq("foo")
rec.other.should be(rec)
end

it "parses recursive nilable (2)" do
yaml = <<-YAML
--- &1
name: foo
YAML

rec = YAMLRecursiveNilable.from_yaml(yaml)
rec.name.should eq("foo")
rec.other.should be_nil
end

it "parses recursive array" do
yaml = <<-YAML
---
name: foo
other: &1
- name: bar
other: *1
YAML

rec = YAMLRecursiveArray.from_yaml(yaml)
rec.other[0].other.should be(rec.other)
end

it "parses recursive hash" do
yaml = <<-YAML
---
name: foo
other: &1
foo:
name: bar
other: *1
YAML

rec = YAMLRecursiveHash.from_yaml(yaml)
rec.other["foo"].other.should be(rec.other)
end

describe "parses YAML with defaults" do
it "mixed" do
json = YAMLWithDefaults.from_yaml(%({"a":1,"b":"bla"}))
@@ -214,11 +333,22 @@ describe "YAML mapping" do
json = YAMLWithDefaults.from_yaml(%({}))
json.a.should eq 11
json.b.should eq "Haha"
end

it "mixes with all defaults (#2873)" do
yaml = YAMLWithDefaults.from_yaml("")
yaml.a.should eq 11
yaml.b.should eq "Haha"
end

it "raises when not a mapping or empty scalar" do
expect_raises(YAML::ParseException) do
YAMLWithDefaults.from_yaml("1")
end

# There's no "null" in YAML? Maybe we should support this eventually
# json = YAMLWithDefaults.from_yaml(%({"a":null,"b":null}))
# json.a.should eq 11
# json.b.should eq "Haha"
expect_raises(YAML::ParseException) do
YAMLWithDefaults.from_yaml("[1]")
end
end

it "bool" do
202 changes: 202 additions & 0 deletions spec/std/yaml/schema/core_spec.cr
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,202 @@
require "spec"
require "yaml"

private def it_parses(string, expected, file = __FILE__, line = __LINE__)
it "parses #{string.inspect}", file, line do
YAML::Schema::Core.parse(string).should eq(expected)
end
end

private def it_raises_on_parse(string, message, file = __FILE__, line = __LINE__)
it "raises on parse #{string.inspect}", file, line do
expect_raises(YAML::ParseException, message) do
YAML::Schema::Core.parse(string)
end
end
end

private def it_parses_scalar(string, expected, file = __FILE__, line = __LINE__)
it "parses #{string.inspect}", file, line do
YAML::Schema::Core.parse_scalar(string).should eq(expected)
end
end

private def it_parses_string(string, file = __FILE__, line = __LINE__)
it_parses_scalar(string, string, file, line)
end

private def it_parses_scalar_from_pull(string, expected, file = __FILE__, line = __LINE__)
it_parses_scalar_from_pull(string, file, line) do |value|
value.should eq(expected)
end
end

private def it_parses_scalar_from_pull(string, file = __FILE__, line = __LINE__, &block : YAML::Type ->)
it "parses #{string.inspect}", file, line do
pull = YAML::PullParser.new(%(value: #{string}))
pull.read_stream_start
pull.read_document_start
pull.read_mapping_start
pull.read_scalar # key

block.call(YAML::Schema::Core.parse_scalar(pull).as(YAML::Type))
end
end

describe YAML::Schema::Core do
# nil
it_parses_scalar "~", nil
it_parses_scalar "null", nil
it_parses_scalar "Null", nil
it_parses_scalar "NULL", nil

# true
it_parses_scalar "yes", true
it_parses_scalar "Yes", true
it_parses_scalar "YES", true
it_parses_scalar "true", true
it_parses_scalar "True", true
it_parses_scalar "TRUE", true
it_parses_scalar "on", true
it_parses_scalar "On", true
it_parses_scalar "ON", true

# false
it_parses_scalar "no", false
it_parses_scalar "No", false
it_parses_scalar "NO", false
it_parses_scalar "false", false
it_parses_scalar "False", false
it_parses_scalar "FALSE", false
it_parses_scalar "off", false
it_parses_scalar "Off", false
it_parses_scalar "OFF", false

# +infinity
it_parses_scalar ".inf", Float64::INFINITY
it_parses_scalar ".Inf", Float64::INFINITY
it_parses_scalar ".INF", Float64::INFINITY
it_parses_scalar "+.inf", Float64::INFINITY
it_parses_scalar "+.Inf", Float64::INFINITY
it_parses_scalar "+.INF", Float64::INFINITY

# -infinity
it_parses_scalar "-.inf", -Float64::INFINITY
it_parses_scalar "-.Inf", -Float64::INFINITY
it_parses_scalar "-.INF", -Float64::INFINITY

# nan
it "parses nan" do
{".nan", ".NaN", ".NAN"}.each do |string|
value = YAML::Schema::Core.parse_scalar(string)
value.as(Float64).nan?.should be_true
end
end

# integer (base 10)
it_parses_scalar "123", 123
it_parses_scalar "+123", 123
it_parses_scalar "-123", -123

# integer (binary)
it_parses_scalar "0b10110", 0b10110

# integer (octal)
it_parses_scalar "0123", 0o123

# integer (hex)
it_parses_scalar "0x123abc", 0x123abc
it_parses_scalar "-0x123abc", -0x123abc

# time
it_parses_scalar "2002-12-14", Time.new(2002, 12, 14, kind: Time::Kind::Utc)
it_parses_scalar "2002-1-2", Time.new(2002, 1, 2, kind: Time::Kind::Utc)
it_parses_scalar "2002-1-2T10:11:12", Time.new(2002, 1, 2, 10, 11, 12, kind: Time::Kind::Utc)
it_parses_scalar "2002-1-2 10:11:12", Time.new(2002, 1, 2, 10, 11, 12, kind: Time::Kind::Utc)
it_parses_scalar "2002-1-2 1:11:12", Time.new(2002, 1, 2, 1, 11, 12, kind: Time::Kind::Utc)
it_parses_scalar "2002-1-2T10:11:12.3", Time.new(2002, 1, 2, 10, 11, 12, 300, kind: Time::Kind::Utc)
it_parses_scalar "2002-1-2T10:11:12.34", Time.new(2002, 1, 2, 10, 11, 12, 340, kind: Time::Kind::Utc)
it_parses_scalar "2002-1-2T10:11:12.345", Time.new(2002, 1, 2, 10, 11, 12, 345, kind: Time::Kind::Utc)
it_parses_scalar "2002-1-2T10:11:12.3456", Time.new(2002, 1, 2, 10, 11, 12, 345, kind: Time::Kind::Utc)
it_parses_scalar "2002-1-2T10:11:12Z", Time.new(2002, 1, 2, 10, 11, 12, kind: Time::Kind::Utc)
it_parses_scalar "2002-1-2T10:11:12 Z", Time.new(2002, 1, 2, 10, 11, 12, kind: Time::Kind::Utc)
it_parses_scalar "2002-1-2T10:11:12 +3", Time.new(2002, 1, 2, 7, 11, 12, kind: Time::Kind::Utc)
it_parses_scalar "2002-1-2T10:11:12 +03:00", Time.new(2002, 1, 2, 7, 11, 12, kind: Time::Kind::Utc)
it_parses_scalar "2002-1-2T10:11:12 -03:00", Time.new(2002, 1, 2, 13, 11, 12, kind: Time::Kind::Utc)
it_parses_scalar "2002-1-2T10:11:12 -03:31", Time.new(2002, 1, 2, 13, 42, 12, kind: Time::Kind::Utc)
it_parses_scalar "2002-1-2T10:11:12-03:31", Time.new(2002, 1, 2, 13, 42, 12, kind: Time::Kind::Utc)
it_parses_scalar "2002-1-2T10:11:12 +0300", Time.new(2002, 1, 2, 7, 11, 12, kind: Time::Kind::Utc)

# invalid time
it_parses_string "2002-34-45"
it_parses_string "2002-12-14 x"
it_parses_string "2002-1-2T10:11:12x"
it_parses_string "2002-1-2T10:11:12Zx"
it_parses_string "2002-1-2T10:11:12+03x"

# non-plain style
it_parses_scalar_from_pull %("1"), "1"

# bools according to the spec, but parsed as strings in Python and Ruby,
# so we do the same in Crystal for "compatibility"
it_parses_scalar "y", "y"
it_parses_scalar "Y", "Y"
it_parses_scalar "n", "n"
it_parses_scalar "N", "N"

# !!map
it_parses "!!map {1: 2}", {1 => 2}
it_raises_on_parse "!!map 1", "Expected MAPPING_START"

# !!omap
it_parses "!!omap {1: 2}", {1 => 2}
it_raises_on_parse "!!omap 1", "Expected MAPPING_START"

# !!pairs
it_parses "!!pairs [{1: 2}, {3: 4}]", [{1 => 2}, {3 => 4}]
it_raises_on_parse "!!pairs 1", "Expected SEQUENCE_START"
it_raises_on_parse "!!pairs [{1: 2, 3: 4}]", "Expected MAPPING_END"

# !!set
it_parses "!!set { 1, 2, 3 }", Set{1, 2, 3}
it_raises_on_parse "!!set 1", "Expected MAPPING_START"

# !!seq
it_parses "!!seq [ 1, 2, 3 ]", [1, 2, 3]
it_raises_on_parse "!!seq 1", "Expected SEQUENCE_START"

# !!binary
it_parses "!!binary aGVsbG8=", "hello".to_slice
it_raises_on_parse "!!binary [1]", "Expected SCALAR"
it_raises_on_parse "!!binary 1", "Error decoding Base64"

# !!bool
it_parses "!!bool yes", true
it_raises_on_parse "!!bool 1", "Invalid bool"

# !!float
it_parses "!!float '1.2'", 1.2
it_parses "!!float '1_234.2'", 1_234.2
it_parses "!!float .inf", Float64::INFINITY
it_raises_on_parse "!!float 'hello'", "Invalid float"

# !!int
it_parses "!!int 123", 123
it_parses "!!int 0b10", 0b10
it_parses "!!int 0123", 0o123
it_parses "!!int 0xabc", 0xabc
it_parses "!!int -123", -123
it_raises_on_parse "!!int 'hello'", "Invalid int"

# !!null
it_parses "!!null ~", nil
it_raises_on_parse "!!null 1", "Invalid null"

# !!str
it_parses "!!str 1", "1"
it_raises_on_parse "!!str [1]", "Expected SCALAR"

# # !!timestamp
it_parses "!!timestamp 2010-01-02", Time.new(2010, 1, 2, kind: Time::Kind::Utc)
it_raises_on_parse "!!timestamp foo", "Invalid timestamp"
end
111 changes: 98 additions & 13 deletions spec/std/yaml/serialization_spec.cr
View file
Original file line number Diff line number Diff line change
@@ -9,19 +9,31 @@ enum YAMLSpecEnum
Two
end

alias YamlRec = Int32 | Array(YamlRec) | Hash(YamlRec, YamlRec)

describe "YAML serialization" do
describe "from_yaml" do
it "does Nil#from_yaml" do
%w(~ null Null NULL).each do |string|
Nil.from_yaml(string).should be_nil
end
Nil.from_yaml("--- \n...\n").should be_nil
end

it "does Bool#from_yaml" do
Bool.from_yaml("true").should be_true
Bool.from_yaml("false").should be_false
%w(yes Yes YES true True TRUE on On ON).each do |string|
Bool.from_yaml(string).should be_true
end

%w(no No NO false False FALSE off Off OFF).each do |string|
Bool.from_yaml(string).should be_false
end
end

it "does Int32#from_yaml" do
Int32.from_yaml("123").should eq(123)
Int32.from_yaml("0xabc").should eq(0xabc)
Int32.from_yaml("0b10110").should eq(0b10110)
end

it "does Int64#from_yaml" do
@@ -32,8 +44,15 @@ describe "YAML serialization" do
String.from_yaml("hello").should eq("hello")
end

it "raises on reserved string" do
expect_raises(YAML::ParseException) do
String.from_yaml(%(1.2))
end
end

it "does Float32#from_yaml" do
Float32.from_yaml("1.5").should eq(1.5)
Float32.from_yaml("1.5").should eq(1.5_f32)
Float32.from_yaml(".inf").should eq(Float32::INFINITY)
end

it "does Float64#from_yaml" do
@@ -63,6 +82,35 @@ describe "YAML serialization" do
Hash(Int32, Bool).from_yaml("---\n1: true\n2: false\n").should eq({1 => true, 2 => false})
end

it "does Hash#from_yaml with merge" do
yaml = <<-YAML
- &foo
bar: 1
baz: 2
-
<<: *foo
YAML

array = Array(Hash(String, Int32)).from_yaml(yaml)
array[1].should eq(array[0])
end

it "does Hash#from_yaml with merge (recursive)" do
yaml = <<-YAML
- &foo
foo: 1
- &bar
bar: 2
<<: *foo
-
<<: *bar
YAML

array = Array(Hash(String, Int32)).from_yaml(yaml)
array[2].should eq({"foo" => 1, "bar" => 2})
end

it "does Tuple#from_yaml" do
Tuple(Int32, String, Bool).from_yaml("---\n- 1\n- foo\n- true\n").should eq({1, "foo", true})
end
@@ -102,20 +150,22 @@ describe "YAML serialization" do
end

it "does Time::Format#from_yaml" do
pull = YAML::PullParser.new("--- 2014-01-02\n...\n")
pull.read_stream do
pull.read_document do
Time::Format.new("%F").from_yaml(pull).should eq(Time.new(2014, 1, 2))
end
end
ctx = YAML::ParseContext.new
nodes = YAML::Nodes.parse("--- 2014-01-02\n...\n").nodes.first
value = Time::Format.new("%F").from_yaml(ctx, nodes)
value.should eq(Time.new(2014, 1, 2))
end

it "deserializes union" do
Array(Int32 | String).from_yaml(%([1, "hello"])).should eq([1, "hello"])
end

it "deserializes time" do
Time.from_yaml(%(2016-11-16T09:55:48-0300)).to_utc.should eq(Time.new(2016, 11, 16, 12, 55, 48, kind: Time::Kind::Utc))
Time.from_yaml("2010-11-12").should eq(Time.new(2010, 11, 12, kind: Time::Kind::Utc))
end

it "deserializes bytes" do
Bytes.from_yaml("!!binary aGVsbG8=").should eq("hello".to_slice)
end

describe "parse exceptions" do
@@ -127,7 +177,7 @@ describe "YAML serialization" do
]
YAML
end
ex.message.should eq("Expected nil, not 1 at line 2, column 3")
ex.message.should eq("Expected Nil, not 1 at line 2, column 3")
ex.location.should eq({2, 3})
end

@@ -200,6 +250,12 @@ describe "YAML serialization" do
String.from_yaml("hel\\lo".to_yaml).should eq("hel\\lo")
end

it "quotes string if reserved" do
["1", "1.2", "true", "2010-11-12"].each do |string|
string.to_yaml.should eq(%(--- "#{string}"\n))
end
end

it "does for Array" do
Array(Int32).from_yaml([1, 2, 3].to_yaml).should eq([1, 2, 3])
end
@@ -238,8 +294,23 @@ describe "YAML serialization" do
YAMLSpecEnum.from_yaml(YAMLSpecEnum::One.to_yaml).should eq(YAMLSpecEnum::One)
end

it "does for time" do
Time.new(2016, 11, 16, 12, 55, 48, kind: Time::Kind::Utc).to_yaml.should eq("--- 2016-11-16T12:55:48+0000\n...\n")
it "does for utc time" do
time = Time.new(2010, 11, 12, 1, 2, 3, kind: Time::Kind::Utc)
time.to_yaml.should eq("--- 2010-11-12 01:02:03\n...\n")
end

it "does for time at date" do
time = Time.new(2010, 11, 12, kind: Time::Kind::Utc)
time.to_yaml.should eq("--- 2010-11-12\n...\n")
end

it "does for utc time with milliseconds" do
time = Time.new(2010, 11, 12, 1, 2, 3, 456, kind: Time::Kind::Utc)
time.to_yaml.should eq("--- 2010-11-12 01:02:03.456\n...\n")
end

it "does for bytes" do
"hello".to_slice.to_yaml.should eq("--- !!binary 'aGVsbG8=\n\n'\n")
end

it "does a full document" do
@@ -266,5 +337,19 @@ describe "YAML serialization" do
end
string.should eq("---\n- a\n- b\n- c\n")
end

it "serializes recursive data structures" do
a = [] of YamlRec
a << 1
a << a

a.to_yaml.should eq("--- &1\n- 1\n- *1\n")

h = {} of YamlRec => YamlRec
h[1] = 2
h[h] = h

h.to_yaml.should eq("--- &1\n1: 2\n*1: *1\n")
end
end
end
17 changes: 0 additions & 17 deletions spec/std/yaml/yaml_pull_parser_spec.cr
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,6 @@
require "spec"
require "yaml"

private def assert_raw(string, expected = string, file = __FILE__, line = __LINE__)
it "parses raw #{string.inspect}", file, line do
pull = YAML::PullParser.new(string)
pull.read_stream do
pull.read_document do
pull.read_raw.should eq(expected)
end
end
end
end

module YAML
describe PullParser do
it "reads empty stream" do
@@ -116,12 +105,6 @@ module YAML
end
end

assert_raw %(hello)
assert_raw %("hello"), %(hello)
assert_raw %(["hello"])
assert_raw %(["hello","world"])
assert_raw %({"hello":"world"})

it "raises exception at correct location" do
parser = PullParser.new("[1]")
parser.read_stream do
39 changes: 31 additions & 8 deletions spec/std/yaml/yaml_spec.cr
View file
Original file line number Diff line number Diff line change
@@ -8,25 +8,25 @@ describe "YAML" do
it { YAML.parse_all("---\nfoo\n---\nbar\n").should eq(["foo", "bar"]) }
it { YAML.parse("foo: bar").should eq({"foo" => "bar"}) }
it { YAML.parse("--- []\n").should eq([] of YAML::Type) }
it { YAML.parse("---\n...").should eq("") }
it { YAML.parse("---\n...").should be_nil }

it "parses recursive sequence" do
doc = YAML.parse("--- &foo\n- *foo\n")
doc[0].raw.should be(doc.raw)
doc[0].raw.as(Array).should be(doc.raw.as(Array))
end

it "parses recursive mapping" do
doc = YAML.parse(%(--- &1
friends:
- *1
))
doc["friends"][0].raw.should be(doc.raw)
doc["friends"][0].raw.as(Hash).should be(doc.raw.as(Hash))
end

it "parses alias to scalar" do
doc = YAML.parse("---\n- &x foo\n- *x\n")
doc.should eq(["foo", "foo"])
doc[0].raw.should be(doc[1].raw)
doc[0].raw.as(String).should be(doc[1].raw.as(String))
end

describe "merging with << key" do
@@ -48,6 +48,29 @@ describe "YAML" do
end
end

it "merges other mapping with alias" do
doc = YAML.parse(%(---
foo: &x
bar: 1
baz: 2
bar:
<<: *x
))
doc["bar"].should eq({"bar" => 1, "baz" => 2})
end

it "merges other mapping with array of alias" do
doc = YAML.parse(%(---
foo: &x
bar: 1
bar: &y
baz: 2
bar:
<<: [*x, *y]
))
doc["bar"].should eq({"bar" => 1, "baz" => 2})
end

it "doesn't merge explicit string key <<" do
doc = YAML.parse(%(---
foo: &foo
@@ -64,7 +87,7 @@ describe "YAML" do
bar:
<<: *foo
))
doc["bar"].should eq({"<<" => ""})
doc["bar"].should eq({"<<" => nil})
end

it "doesn't merge arrays" do
@@ -74,7 +97,7 @@ describe "YAML" do
bar:
<<: *foo
))
doc["bar"].should eq({"<<" => ["1"]})
doc["bar"].should eq({"<<" => [1]})
end

it "has correct line/number info (#2585)" do
@@ -130,14 +153,14 @@ describe "YAML" do

describe "dump" do
it "returns YAML as a string" do
YAML.dump(%w(1 2 3)).should eq("---\n- 1\n- 2\n- 3\n")
YAML.dump(%w(1 2 3)).should eq(%(---\n- "1"\n- "2"\n- "3"\n))
end

it "writes YAML to a stream" do
string = String.build do |str|
YAML.dump(%w(1 2 3), str)
end
string.should eq("---\n- 1\n- 2\n- 3\n")
string.should eq(%(---\n- "1"\n- "2"\n- "3"\n))
end
end
end
16 changes: 12 additions & 4 deletions src/big/yaml.cr
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,18 @@
require "yaml"
require "big"

def BigInt.new(pull : YAML::PullParser)
BigInt.new(pull.read_scalar)
def BigInt.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node)
unless node.is_a?(YAML::Nodes::Scalar)
node.raise "Expected scalar, not #{node.class}"
end

BigInt.new(node.value)
end

def BigFloat.new(pull : YAML::PullParser)
BigFloat.new(pull.read_scalar)
def BigFloat.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node)
unless node.is_a?(YAML::Nodes::Scalar)
node.raise "Expected scalar, not #{node.class}"
end

BigFloat.new(node.value)
end
68 changes: 33 additions & 35 deletions src/yaml.cr
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,15 @@
require "./yaml/*"
require "./yaml/**"
require "base64"

# The YAML module provides serialization and deserialization of YAML to/from native Crystal data structures.
# The YAML module provides serialization and deserialization of YAML
# version 1.1 to/from native Crystal data structures, with the additional
# independent types specified in http://yaml.org/type/
#
# ### Parsing with `#parse` and `#parse_all`
#
# `YAML#parse` will return an `Any`, which is a convenient wrapper around all possible YAML types,
# making it easy to traverse a complex YAML structure but requires some casts from time to time,
# mostly via some method invocations.
# `YAML#parse` will return an `Any`, which is a convenient wrapper around all possible
# YAML core types, making it easy to traverse a complex YAML structure but requires
# some casts from time to time, mostly via some method invocations.
#
# ```
# require "yaml"
@@ -22,35 +25,31 @@ require "./yaml/*"
# data["foo"]["bar"]["baz"][1].as_s # => "fox"
# ```
#
# ### Parsing with `YAML#mapping`
# ### Parsing with `from_yaml`
#
# `YAML#mapping` defines how an object is mapped to YAML. Mapped data is accessible
# through generated properties like *Foo#bar*. It is more type-safe and efficient.
# A type `T` can be deserialized from YAML by invoking `T.from_yaml(string_or_io)`.
# For this to work, `T` must implement
# `new(ctx : YAML::PullParser, node : YAML::Nodes::Node)` and decode
# a value from the given *node*, using *ctx* to store and retrieve
# anchored values (see `YAML::PullParser` for an explanation of this).
#
# ### Generating with `YAML.build`
# Crystal primitive types, `Time`, `Bytes` and `Union` implement
# this method. `YAML.mapping` can be used to implement this method
# for user types.
#
# Use `YAML.build`, which uses `YAML::Builder`, to generate YAML
# by emitting scalars, sequences and mappings:
#
# ```
# require "yaml"
# ### Dumping with `YAML.dump` or `#to_yaml`
#
# string = YAML.build do |yaml|
# yaml.mapping do
# yaml.scalar "foo"
# yaml.sequence do
# yaml.scalar 1
# yaml.scalar 2
# end
# end
# end
# string # => "---\nfoo:\n- 1\n- 2\n"
# ```
# `YAML.dump` generates the YAML representation for an object.
# An `IO` can be passed and it will be written there,
# otherwise it will be returned as a string. Similarly, `#to_yaml`
# (with or without an `IO`) on any object does the same.
#
# ### Dumping with `YAML.dump` or `#to_yaml`
# For this to work, the type given to `YAML.dump` must implement
# `to_yaml(builder : YAML::Nodes::Builder`).
#
# `YAML.dump` generates the YAML representation for an object. An `IO` can be passed and it will be written there,
# otherwise it will be returned as a string. Similarly, `#to_yaml` (with or without an `IO`) on any object does the same.
# Crystal primitive types, `Time` and `Bytes` implement
# this method. `YAML.mapping` can be used to implement this method
# for user types.
#
# ```
# yaml = YAML.dump({hello: "world"}) # => "---\nhello: world\n"
@@ -84,11 +83,10 @@ module YAML
end
end

# All valid YAML types.
alias Type = String | Hash(Type, Type) | Array(Type) | Nil
alias EventKind = LibYAML::EventType
# All valid YAML core schema types.
alias Type = Nil | Bool | Int64 | Float64 | String | Time | Bytes | Array(Type) | Hash(Type, Type) | Set(Type)

# Deserializes a YAML document.
# Deserializes a YAML document according to the core schema.
#
# ```yaml
# # ./foo.yml
@@ -116,10 +114,10 @@ module YAML
# # => }
# ```
def self.parse(data : String | IO) : Any
YAML::Parser.new data, &.parse
YAML::Schema::Core.parse(data)
end

# Deserializes multiple YAML documents.
# Deserializes multiple YAML documents according to the core schema.
#
# ```yaml
# # ./foo.yml
@@ -135,7 +133,7 @@ module YAML
# # => [{"foo" => "bar"}, {"hello" => "world"}]
# ```
def self.parse_all(data : String) : Array(Any)
YAML::Parser.new data, &.parse_all
YAML::Schema::Core.parse_all(data)
end

# Serializes an object to YAML, returning it as a `String`.
185 changes: 123 additions & 62 deletions src/yaml/any.cr
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
# `YAML::Any` is a convenient wrapper around all possible YAML types (`YAML::Type`)
# and can be used for traversing dynamic or unknown YAML structures.
# `YAML::Any` is a convenient wrapper around all possible YAML core types
# (`YAML::Type`) and can be used for traversing dynamic or
# unknown YAML structures.
#
# ```
# require "yaml"
@@ -25,29 +26,43 @@
struct YAML::Any
include Enumerable(self)

# Reads a `YAML::Any` value from the given pull parser.
def self.new(pull : YAML::PullParser)
case pull.kind
when .scalar?
new pull.read_scalar
when .sequence_start?
def self.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node)
anchors = {} of String => Type
new(convert(node, anchors))
end

private def self.convert(node, anchors)
case node
when YAML::Nodes::Scalar
YAML::Schema::Core.parse_scalar(node.value)
when YAML::Nodes::Sequence
ary = [] of Type
pull.read_sequence do
while pull.kind != YAML::EventKind::SEQUENCE_END
ary << new(pull).raw
end

if anchor = node.anchor
anchors[anchor] = ary
end

node.each do |value|
ary << convert(value, anchors)
end
new ary
when .mapping_start?

ary
when YAML::Nodes::Mapping
hash = {} of Type => Type
pull.read_mapping do
while pull.kind != YAML::EventKind::MAPPING_END
hash[new(pull).raw] = new(pull).raw
end

if anchor = node.anchor
anchors[anchor] = hash
end
new hash

node.each do |key, value|
hash[convert(key, anchors)] = convert(value, anchors)
end

hash
when YAML::Nodes::Alias
anchors[node.anchor]
else
raise "Unknown pull kind: #{pull.kind}"
raise "Unknown node: #{node.class}"
end
end

@@ -72,57 +87,43 @@ struct YAML::Any
end
end

# Assumes the underlying value is an `Array` and returns the element
# at the given *index*.
# Assumes the underlying value is an `Array` or `Hash`
# and returns the element at the given *index_or_key*.
#
# Raises if the underlying value is not an `Array`.
def [](index : Int) : YAML::Any
# Raises if the underlying value is not an `Array` nor a `Hash`.
def [](index_or_key) : YAML::Any
case object = @raw
when Array
Any.new object[index]
else
raise "Expected Array for #[](index : Int), not #{object.class}"
end
end

# Assumes the underlying value is an `Array` and returns the element
# at the given *index*, or `nil` if out of bounds.
#
# Raises if the underlying value is not an `Array`.
def []?(index : Int) : YAML::Any?
case object = @raw
when Array
value = object[index]?
value ? Any.new(value) : nil
else
raise "Expected Array for #[]?(index : Int), not #{object.class}"
end
end

# Assumes the underlying value is a `Hash` and returns the element
# with the given *key*.
#
# Raises if the underlying value is not a `Hash`.
def [](key : String) : YAML::Any
case object = @raw
if index_or_key.is_a?(Int)
Any.new object[index_or_key]
else
raise "Expected int key for Array#[], not #{object.class}"
end
when Hash
Any.new object[key]
Any.new object[index_or_key]
else
raise "Expected Hash for #[](key : String), not #{object.class}"
raise "Expected Array or Hash, not #{object.class}"
end
end

# Assumes the underlying value is a `Hash` and returns the element
# with the given *key*, or `nil` if the key is not present.
# Assumes the underlying value is an `Array` or `Hash` and returns the element
# at the given *index_or_key*, or `nil` if out of bounds or the key is missing.
#
# Raises if the underlying value is not a `Hash`.
def []?(key : String) : YAML::Any?
# Raises if the underlying value is not an `Array` nor a `Hash`.
def []?(index_or_key) : YAML::Any?
case object = @raw
when Array
if index_or_key.is_a?(Int)
value = object[index_or_key]?
value ? Any.new(value) : nil
else
nil
end
when Hash
value = object[key]?
value = object[index_or_key]?
value ? Any.new(value) : nil
else
raise "Expected Hash for #[]?(key : String), not #{object.class}"
raise "Expected Array or Hash, not #{object.class}"
end
end

@@ -141,7 +142,7 @@ struct YAML::Any
yield Any.new(key), Any.new(value)
end
else
raise "Expected Array or Hash for #each, not #{object.class}"
raise "Expected Array or Hash, not #{object.class}"
end
end

@@ -160,7 +161,55 @@ struct YAML::Any
# Checks that the underlying value is `String`, and returns its value.
# Returns `nil` otherwise.
def as_s? : String?
as_s if @raw.is_a?(String)
@raw.as?(String)
end

# Checks that the underlying value is `Int64`, and returns its value.
# Raises otherwise.
def as_i64 : Int64
@raw.as(Int64)
end

# Checks that the underlying value is `Int64`, and returns its value.
# Returns `nil` otherwise.
def as_i64? : Int64?
@raw.as?(Int64)
end

# Checks that the underlying value is `Int64`, and returns its value as `Int32`.
# Raises otherwise.
def as_i : Int32
@raw.as(Int64).to_i
end

# Checks that the underlying value is `Int64`, and returns its value as `Int32`.
# Returns `nil` otherwise.
def as_i? : Int32?
@raw.as?(Int64).try &.to_i
end

# Checks that the underlying value is `Float64`, and returns its value.
# Raises otherwise.
def as_f : Float64
@raw.as(Float64)
end

# Checks that the underlying value is `Float64`, and returns its value.
# Returns `nil` otherwise.
def as_f? : Float64?
@raw.as?(Float64)
end

# Checks that the underlying value is `Time`, and returns its value.
# Raises otherwise.
def as_time : Time
@raw.as(Time)
end

# Checks that the underlying value is `Time`, and returns its value.
# Returns `nil` otherwise.
def as_time? : Time?
@raw.as?(Time)
end

# Checks that the underlying value is `Array`, and returns its value.
@@ -172,7 +221,7 @@ struct YAML::Any
# Checks that the underlying value is `Array`, and returns its value.
# Returns `nil` otherwise.
def as_a? : Array(Type)?
as_a if @raw.is_a?(Array(Type))
@raw.as?(Array)
end

# Checks that the underlying value is `Hash`, and returns its value.
@@ -184,7 +233,19 @@ struct YAML::Any
# Checks that the underlying value is `Hash`, and returns its value.
# Returns `nil` otherwise.
def as_h? : Hash(Type, Type)?
as_h if @raw.is_a?(Hash(Type, Type))
@raw.as?(Hash)
end

# Checks that the underlying value is `Bytes`, and returns its value.
# Raises otherwise.
def as_bytes : Bytes
@raw.as(Bytes)
end

# Checks that the underlying value is `Bytes`, and returns its value.
# Returns `nil` otherwise.
def as_bytes? : Bytes?
@raw.as?(Bytes)
end

# :nodoc:
51 changes: 41 additions & 10 deletions src/yaml/builder.cr
View file
Original file line number Diff line number Diff line change
@@ -3,6 +3,21 @@
# A `YAML::Error` is raised if attempting to generate
# an invalid YAML (for example, if invoking `end_sequence`
# without a matching `start_sequence`)
#
# ```
# require "yaml"
#
# string = YAML.build do |yaml|
# yaml.mapping do
# yaml.scalar "foo"
# yaml.sequence do
# yaml.scalar 1
# yaml.scalar 2
# end
# end
# end
# string # => "---\nfoo:\n- 1\n- 2\n"
# ```
class YAML::Builder
@box : Void*

@@ -60,14 +75,16 @@ class YAML::Builder
end

# Emits a scalar value.
def scalar(value)
def scalar(value, anchor : String? = nil, tag : String? = nil, style : YAML::ScalarStyle = YAML::ScalarStyle::ANY)
string = value.to_s
emit scalar, nil, nil, string, string.bytesize, 1, 1, LibYAML::ScalarStyle::ANY
implicit = tag ? 0 : 1
emit scalar, get_anchor(anchor), string_to_unsafe(tag), string, string.bytesize, implicit, implicit, style
end

# Starts a sequence.
def start_sequence
emit sequence_start, nil, nil, 0, LibYAML::SequenceStyle::ANY
def start_sequence(anchor : String? = nil, tag : String? = nil, style : YAML::SequenceStyle = YAML::SequenceStyle::ANY)
implicit = tag ? 0 : 1
emit sequence_start, get_anchor(anchor), string_to_unsafe(tag), implicit, style
end

# Ends a sequence.
@@ -76,14 +93,15 @@ class YAML::Builder
end

# Starts a sequence, invokes the block, and the ends it.
def sequence
start_sequence
def sequence(anchor : String? = nil, tag : String? = nil, style : YAML::SequenceStyle = YAML::SequenceStyle::ANY)
start_sequence(anchor, tag, style)
yield.tap { end_sequence }
end

# Starts a mapping.
def start_mapping
emit mapping_start, nil, nil, 0, LibYAML::MappingStyle::ANY
def start_mapping(anchor : String? = nil, tag : String? = nil, style : YAML::MappingStyle = YAML::MappingStyle::ANY)
implicit = tag ? 0 : 1
emit mapping_start, get_anchor(anchor), string_to_unsafe(tag), implicit, style
end

# Ends a mapping.
@@ -92,11 +110,16 @@ class YAML::Builder
end

# Starts a mapping, invokes the block, and then ends it.
def mapping
start_mapping
def mapping(anchor : String? = nil, tag : String? = nil, style : YAML::MappingStyle = YAML::MappingStyle::ANY)
start_mapping(anchor, tag, style)
yield.tap { end_mapping }
end

def alias(anchor : String)
LibYAML.yaml_alias_event_initialize(pointerof(@event), anchor)
yaml_emit("alias")
end

# Flushes any pending data to the underlying `IO`.
def flush
LibYAML.yaml_emitter_flush(@emitter)
@@ -113,6 +136,14 @@ class YAML::Builder
@closed = true
end

private def get_anchor(anchor)
string_to_unsafe(anchor)
end

private def string_to_unsafe(tag)
tag.try(&.to_unsafe) || Pointer(UInt8).null
end

private macro emit(event_name, *args)
LibYAML.yaml_{{event_name}}_event_initialize(pointerof(@event), {{*args}})
yaml_emit({{event_name.stringify}})
36 changes: 36 additions & 0 deletions src/yaml/enums.cr
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
module YAML
enum EventKind
NONE
STREAM_START
STREAM_END
DOCUMENT_START
DOCUMENT_END
ALIAS
SCALAR
SEQUENCE_START
SEQUENCE_END
MAPPING_START
MAPPING_END
end

enum ScalarStyle
ANY
PLAIN
SINGLE_QUOTED
DOUBLE_QUOTED
LITERAL
FOLDED
end

enum SequenceStyle
ANY
BLOCK
FLOW
end

enum MappingStyle
ANY
BLOCK
FLOW
end
end
232 changes: 148 additions & 84 deletions src/yaml/from_yaml.cr
View file
Original file line number Diff line number Diff line change
@@ -1,132 +1,160 @@
def Object.from_yaml(string_or_io) : self
YAML::PullParser.new(string_or_io) do |parser|
parser.read_stream do
parser.read_document do
new parser
end
end
def Object.from_yaml(string_or_io : String | IO) : self
new(YAML::ParseContext.new, parse_yaml(string_or_io))
end

def Array.from_yaml(string_or_io : String | IO)
new(YAML::ParseContext.new, parse_yaml(string_or_io)) do |element|
yield element
end
end

def Array.from_yaml(string_or_io)
YAML::PullParser.new(string_or_io) do |parser|
parser.read_stream do
parser.read_document do
new(parser) do |element|
yield element
end
end
end
private def parse_yaml(string_or_io)
document = YAML::Nodes.parse(string_or_io)

# If the document is empty we simulate an empty scalar with
# plain style, that parses to Nil
document.nodes.first? || begin
scalar = YAML::Nodes::Scalar.new("")
scalar.style = YAML::ScalarStyle::PLAIN
scalar
end
end

def Nil.new(pull : YAML::PullParser)
location = pull.location
value = pull.read_scalar
if value.empty?
nil
private def parse_scalar(ctx, node, type : T.class) forall T
ctx.read_alias(node, T) do |obj|
return obj
end

if node.is_a?(YAML::Nodes::Scalar)
value = YAML::Schema::Core.parse_scalar(node)
if value.is_a?(T)
ctx.record_anchor(node, value)
value
else
node.raise "Expected #{T}, not #{node.value}"
end
else
raise YAML::ParseException.new("Expected nil, not #{value}", *location)
node.raise "Expected #{T}, not #{node.class.name}"
end
end

def Bool.new(pull : YAML::PullParser)
pull.read_scalar == "true"
def Nil.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node)
parse_scalar(ctx, node, self)
end

def Bool.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node)
parse_scalar(ctx, node, self)
end

{% for type in %w(Int8 Int16 Int32 Int64 UInt8 UInt16 UInt32 UInt64) %}
def {{type.id}}.new(pull : YAML::PullParser)
location = pull.location
begin
{{type.id}}.new(pull.read_scalar)
rescue ex
raise YAML::ParseException.new(ex.message.not_nil!, *location)
end
def {{type.id}}.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node)
{{type.id}}.new parse_scalar(ctx, node, Int64)
end
{% end %}

def String.new(pull : YAML::PullParser)
pull.read_scalar
def String.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node)
parse_scalar(ctx, node, self)
end

def Float32.new(pull : YAML::PullParser)
pull.read_scalar.to_f32
def Float32.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node)
parse_scalar(ctx, node, Float64).to_f32
end

def Float64.new(pull : YAML::PullParser)
pull.read_scalar.to_f64
def Float64.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node)
parse_scalar(ctx, node, self)
end

def Array.new(pull : YAML::PullParser)
def Array.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node)
ctx.read_alias(node, self) do |obj|
return obj
end

ary = new
new(pull) do |element|

ctx.record_anchor(node, ary)

new(ctx, node) do |element|
ary << element
end
ary
end

def Array.new(pull : YAML::PullParser)
pull.read_sequence_start
while pull.kind != YAML::EventKind::SEQUENCE_END
yield T.new(pull)
def Array.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node)
unless node.is_a?(YAML::Nodes::Sequence)
node.raise "Expected sequence, not #{node.class}"
end

node.each do |value|
yield T.new(ctx, value)
end
pull.read_next
end

def Hash.new(pull : YAML::PullParser)
def Hash.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node)
ctx.read_alias(node, self) do |obj|
return obj
end

hash = new
new(pull) do |key, value|

ctx.record_anchor(node, hash)

new(ctx, node) do |key, value|
hash[key] = value
end
hash
end

def Hash.new(pull : YAML::PullParser)
pull.read_mapping_start
while pull.kind != YAML::EventKind::MAPPING_END
yield K.new(pull), V.new(pull)
def Hash.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node)
unless node.is_a?(YAML::Nodes::Mapping)
node.raise "Expected mapping, not #{node.class}"
end

YAML::Schema::Core.each(node) do |key, value|
yield K.new(ctx, key), V.new(ctx, value)
end
pull.read_next
end

def Tuple.new(pull : YAML::PullParser)
def Tuple.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node)
unless node.is_a?(YAML::Nodes::Sequence)
node.raise "Expected sequence, not #{node.class}"
end

if node.nodes.size != {{T.size}}
node.raise "Expected #{{{T.size}}} elements, not #{node.nodes.size}"
end

{% begin %}
pull.read_sequence_start
value = Tuple.new(
Tuple.new(
{% for i in 0...T.size %}
(self[{{i}}].new(pull)),
(self[{{i}}].new(ctx, node.nodes[{{i}}])),
{% end %}
)
pull.read_sequence_end
value
{% end %}
end

def NamedTuple.new(pull : YAML::PullParser)
def NamedTuple.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node)
unless node.is_a?(YAML::Nodes::Mapping)
node.raise "Expected mapping, not #{node.class}"
end

{% begin %}
{% for key in T.keys %}
%var{key.id} = nil
{% end %}

location = pull.location

pull.read_mapping_start
while pull.kind != YAML::EventKind::MAPPING_END
key = pull.read_scalar
YAML::Schema::Core.each(node) do |key, value|
key = String.new(ctx, key)
case key
{% for key, type in T %}
when {{key.stringify}}
%var{key.id} = {{type}}.new(pull)
%var{key.id} = {{type}}.new(ctx, value)
{% end %}
else
pull.skip
end
end
pull.read_mapping_end

{% for key in T.keys %}
if %var{key.id}.nil?
raise YAML::ParseException.new("Missing yaml attribute: {{key}}", *location)
node.raise "Missing yaml attribute: {{key}}"
end
{% end %}

@@ -138,47 +166,83 @@ def NamedTuple.new(pull : YAML::PullParser)
{% end %}
end

def Enum.new(pull : YAML::PullParser)
string = pull.read_scalar
def Enum.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node)
unless node.is_a?(YAML::Nodes::Scalar)
node.raise "Expected scalar, not #{node.class}"
end

string = node.value
if value = string.to_i64?
from_value(value)
else
parse(string)
end
end

def Union.new(pull : YAML::PullParser)
location = pull.location
string = pull.read_raw
def Union.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node)
if node.is_a?(YAML::Nodes::Alias)
{% for type in T %}
{% if type < ::Reference %}
ctx.read_alias?(node, {{type}}) do |obj|
return obj
end
{% end %}
{% end %}

node.raise("Error deserailizing alias")
end

{% for type in T %}
begin
return {{type}}.from_yaml(string)
return {{type}}.new(ctx, node)
rescue YAML::ParseException
# Ignore
end
{% end %}
raise YAML::ParseException.new("Couldn't parse #{self} from #{string}", *location)

node.raise "Couldn't parse #{self}"
end

def Time.new(pull : YAML::PullParser)
Time::Format::ISO_8601_DATE_TIME.parse(pull.read_scalar)
def Time.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node)
parse_scalar(ctx, node, Time)
end

struct Time::Format
def from_yaml(pull : YAML::PullParser)
string = pull.read_scalar
parse(string)
def from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Time
unless node.is_a?(YAML::Nodes::Scalar)
node.raise "Expected scalar, not #{node.class}"
end

parse(node.value)
end
end

module Time::EpochConverter
def self.from_yaml(value : YAML::PullParser) : Time
Time.epoch(value.read_scalar.to_i)
def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Time
unless node.is_a?(YAML::Nodes::Scalar)
node.raise "Expected scalar, not #{node.class}"
end

Time.epoch(node.value.to_i)
end
end

module Time::EpochMillisConverter
def self.from_yaml(value : YAML::PullParser) : Time
Time.epoch_ms(value.read_scalar.to_i64)
def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Time
unless node.is_a?(YAML::Nodes::Scalar)
node.raise "Expected scalar, not #{node.class}"
end

Time.epoch_ms(node.value.to_i64)
end
end

struct Slice
def self.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node)
{% if T != UInt8 %}
{% raise "Can only deserialize Slice(UInt8), not #{@type}}" %}
{% end %}

parse_scalar(ctx, node, self)
end
end
51 changes: 9 additions & 42 deletions src/yaml/lib_yaml.cr
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
require "./enums"

@[Link("yaml")]
lib LibYAML
alias Int = LibC::Int
@@ -42,27 +44,6 @@ lib LibYAML
prefix : UInt8*
end

enum ScalarStyle
ANY
PLAIN
SINGLE_QUOTED
DOUBLE_QUOTED
LITERAL
FOLDED
end

enum SequenceStyle
ANY
BLOCK
FLOW
end

enum MappingStyle
ANY
BLOCK
FLOW
end

struct StreamStartEvent
encoding : Int32
end
@@ -89,21 +70,21 @@ lib LibYAML
length : LibC::SizeT
plain_implicit : Int
quoted_implicit : Int
style : ScalarStyle
style : YAML::ScalarStyle
end

struct SequenceStartEvent
anchor : UInt8*
tag : UInt8*
implicit : Int
style : SequenceStyle
style : YAML::SequenceStyle
end

struct MappingStartEvent
anchor : UInt8*
tag : UInt8*
implicit : Int
style : MappingStyle
style : YAML::MappingStyle
end

union EventData
@@ -116,28 +97,14 @@ lib LibYAML
mapping_start : MappingStartEvent
end

enum EventType
NONE
STREAM_START
STREAM_END
DOCUMENT_START
DOCUMENT_END
ALIAS
SCALAR
SEQUENCE_START
SEQUENCE_END
MAPPING_START
MAPPING_END
end

struct Mark
index : LibC::SizeT
line : LibC::SizeT
column : LibC::SizeT
end

struct Event
type : EventType
type : YAML::EventKind
data : EventData
start_mark : Mark
end_mark : Mark
@@ -167,11 +134,11 @@ lib LibYAML
fun yaml_document_end_event_initialize(event : Event*, implicit : Int) : Int
fun yaml_scalar_event_initialize(event : Event*, anchor : LibC::Char*,
tag : LibC::Char*, value : LibC::Char*, length : Int,
plain_implicit : Int, quoted_implicit : Int, style : ScalarStyle) : Int
plain_implicit : Int, quoted_implicit : Int, style : YAML::ScalarStyle) : Int
fun yaml_alias_event_initialize(event : Event*, anchor : LibC::Char*) : Int
fun yaml_sequence_start_event_initialize(event : Event*, anchor : LibC::Char*, tag : LibC::Char*, implicit : Int, style : SequenceStyle) : Int
fun yaml_sequence_start_event_initialize(event : Event*, anchor : LibC::Char*, tag : LibC::Char*, implicit : Int, style : YAML::SequenceStyle) : Int
fun yaml_sequence_end_event_initialize(event : Event*)
fun yaml_mapping_start_event_initialize(event : Event*, anchor : LibC::Char*, tag : LibC::Char*, implicit : Int, style : MappingStyle) : Int
fun yaml_mapping_start_event_initialize(event : Event*, anchor : LibC::Char*, tag : LibC::Char*, implicit : Int, style : YAML::MappingStyle) : Int
fun yaml_mapping_end_event_initialize(event : Event*) : Int
fun yaml_emitter_emit(emitter : Emitter*, event : Event*) : Int
fun yaml_emitter_delete(emitter : Emitter*)
88 changes: 58 additions & 30 deletions src/yaml/mapping.cr
View file
Original file line number Diff line number Diff line change
@@ -96,50 +96,78 @@ module YAML
{% end %}
{% end %}

def initialize(%pull : ::YAML::PullParser)
def self.new(ctx : YAML::ParseContext, node : YAML::Nodes::Node)
ctx.read_alias(node, \{{@type}}) do |obj|
return obj
end

instance = allocate

ctx.record_anchor(node, instance)

instance.initialize(ctx, node, nil)
instance
end

# `new` and `initialize` with just `pull` as an argument collide
# and the compiler just sees the last one. This is why we add a
# dummy argument.
#
# FIXME: remove the dummy argument if we ever fix this.

def initialize(ctx : YAML::ParseContext, node : ::YAML::Nodes::Node, _dummy : Nil)
{% for key, value in properties %}
%var{key.id} = nil
%found{key.id} = false
{% end %}

%mapping_location = %pull.location
case node
when YAML::Nodes::Mapping
YAML::Schema::Core.each(node) do |key_node, value_node|
unless key_node.is_a?(YAML::Nodes::Scalar)
key_node.raise "Expected scalar as key for mapping"
end

%pull.read_mapping_start
while %pull.kind != ::YAML::EventKind::MAPPING_END
%key_location = %pull.location
key = %pull.read_scalar.not_nil!
case key
{% for key, value in properties %}
when {{value[:key] || key.id.stringify}}
%found{key.id} = true
key = key_node.value

%var{key.id} =
{% if value[:nilable] || value[:default] != nil %} %pull.read_null_or { {% end %}
case key
{% for key, value in properties %}
when {{value[:key] || key.id.stringify}}
%found{key.id} = true

{% if value[:converter] %}
{{value[:converter]}}.from_yaml(%pull)
{% elsif value[:type].is_a?(Path) || value[:type].is_a?(Generic) %}
{{value[:type]}}.new(%pull)
{% else %}
::Union({{value[:type]}}).new(%pull)
{% end %}
%var{key.id} =
{% if value[:nilable] || value[:default] != nil %} YAML::Schema::Core.parse_null_or(value_node) { {% end %}

{% if value[:nilable] || value[:default] != nil %} } {% end %}
{% end %}
else
{% if strict %}
raise ::YAML::ParseException.new("Unknown yaml attribute: #{key}", *%key_location)
{% else %}
%pull.skip
{% if value[:converter] %}
{{value[:converter]}}.from_yaml(ctx, value_node)
{% elsif value[:type].is_a?(Path) || value[:type].is_a?(Generic) %}
{{value[:type]}}.new(ctx, value_node)
{% else %}
::Union({{value[:type]}}).new(ctx, value_node)
{% end %}

{% if value[:nilable] || value[:default] != nil %} } {% end %}
{% end %}
else
{% if strict %}
key_node.raise "Unknown yaml attribute: #{key}"
{% end %}
end
end
when YAML::Nodes::Scalar
if node.value.empty? && node.style.plain? && !node.tag
# We consider an empty scalar as an empty mapping
else
node.raise "Expected mapping, not #{node.class}"
end
else
node.raise "Expected mapping, not #{node.class}"
end
%pull.read_next

{% for key, value in properties %}
{% unless value[:nilable] || value[:default] != nil %}
if %var{key.id}.nil? && !%found{key.id} && !::Union({{value[:type]}}).nilable?
raise ::YAML::ParseException.new("Missing yaml attribute: {{(value[:key] || key).id}}", *%mapping_location)
node.raise "Missing yaml attribute: {{(value[:key] || key).id}}"
end
{% end %}
{% end %}
@@ -165,8 +193,8 @@ module YAML
{% end %}
end

def to_yaml(%yaml : ::YAML::Builder)
%yaml.mapping do
def to_yaml(%yaml : ::YAML::Nodes::Builder)
%yaml.mapping(reference: self) do
{% for key, value in properties %}
_{{key.id}} = @{{key.id}}

15 changes: 15 additions & 0 deletions src/yaml/nodes.cr
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
require "./parser"

# The YAML::Nodes module provides an implementation of an
# in-memory YAML document tree. This tree can be generated
# with the `YAML::Nodes.parse` method or created with a
# `YAML::Nodes::Builder`.
#
# This document tree can then be converted to YAML be
# invoking `to_yaml` on the document object.
module YAML::Nodes
# Parses a `String` or `IO` into a `YAML::Document`.
def self.parse(string_or_io : String | IO) : Document
Parser.new string_or_io, &.parse
end
end
121 changes: 121 additions & 0 deletions src/yaml/nodes/builder.cr
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
# Builds a tree of YAML nodes.
#
# This builder is similar to `YAML::Builder`, but instead of
# directly emitting the output to an IO it builds a YAML document
# tree in memory.
#
# All "emitting" methods support specifying a "reference" object
# that will be associated to the emitted object,
# so that when that reference object is emitted again an anchor
# and an alias will be created. This generates both more compact
# documents and allows handling recursive data structures.
class YAML::Nodes::Builder
@current : Node

# The document this builder builds.
getter document : Document

def initialize
@document = Document.new
@current = @document
@object_id_to_node = {} of UInt64 => Node
@anchor_count = 0
end

def scalar(value, anchor : String? = nil, tag : String? = nil,
style : YAML::ScalarStyle = YAML::ScalarStyle::ANY,
reference = nil) : Nil
node = Scalar.new(value.to_s)
node.anchor = anchor
node.tag = tag
node.style = style

if register(reference, node)
return
end

push_node(node)
end

def sequence(anchor : String? = nil, tag : String? = nil,
style : YAML::SequenceStyle = YAML::SequenceStyle::ANY,
reference = nil) : Nil
node = Sequence.new
node.anchor = anchor
node.tag = tag
node.style = style

if register(reference, node)
return
end

push_to_stack(node) do
yield
end
end

def mapping(anchor : String? = nil, tag : String? = nil,
style : YAML::MappingStyle = YAML::MappingStyle::ANY,
reference = nil) : Nil
node = Mapping.new
node.anchor = anchor
node.tag = tag
node.style = style

if register(reference, node)
return
end

push_to_stack(node) do
yield
end
end

private def push_node(node)
case current = @current
when Document
current << node
when Sequence
current << node
when Mapping
current << node
end
end

private def push_to_stack(node)
push_node(node)

old_current = @current
@current = node

yield

@current = old_current
end

private def register(object, current_node)
if object.is_a?(Reference)
register_object_id(object.object_id, current_node)
else
false
end
end

private def register_object_id(object_id, current_node)
node = @object_id_to_node[object_id]?

if node
anchor = node.anchor ||= begin
@anchor_count += 1
@anchor_count.to_s
end

node = Alias.new(anchor)
push_node(node)
true
else
@object_id_to_node[object_id] = current_node
false
end
end
end
150 changes: 150 additions & 0 deletions src/yaml/nodes/nodes.cr
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,150 @@
module YAML::Nodes
# Abstract class of all YAML tree nodes.
abstract class Node
# The optional tag of a node.
property tag : String?

# The optional anchor of a node.
property anchor : String?

# The line where this node starts.
property start_line = 0

# The column where this node starts.
property start_column = 0

# The line where this node ends.
property end_line = 0

# The column where this node ends.
property end_column = 0

# Returns a tuple of `start_line` and `start_column`.
def location : {Int32, Int32}
{start_line, start_column}
end

# Raises a `YAML::ParseException` with the given message
# located at this node's `location`.
def raise(message)
::raise YAML::ParseException.new(message, *location)
end
end

# A YAML document.
class Document < Node
# The nodes inside this document.
#
# A document can hold at most one node.
getter nodes = [] of Node

# Appends a node to this document. Raises if more
# than one node is appended.
def <<(node)
if nodes.empty?
nodes << node
else
raise ArgumentError.new("Attempted to append more than one node")
end
end

def to_yaml(builder : YAML::Builder)
nodes.each &.to_yaml(builder)
end
end

# A scalar value.
class Scalar < Node
# The style of this scalar.
property style : ScalarStyle = ScalarStyle::ANY

# The value of this scalar.
property value : String

# Creates a scalar with the given *value*.
def initialize(@value : String)
end

def to_yaml(builder : YAML::Builder)
builder.scalar(value, anchor, tag, style)
end
end

# A sequence of nodes.
class Sequence < Node
include Enumerable(Node)

# The nodes in this sequence.
getter nodes = [] of Node

# The style of this sequence.
property style : SequenceStyle = SequenceStyle::ANY

# Appends a node into this sequence.
def <<(node)
@nodes << node
end

def each
@nodes.each do |node|
yield node
end
end

def to_yaml(builder : YAML::Builder)
builder.sequence(anchor, tag, style) do
each &.to_yaml(builder)
end
end
end

# A mapping of nodes.
class Mapping < Node
property style : MappingStyle = MappingStyle::ANY

# The nodes inside this mapping, stored linearly
# as key1 - value1 - key2 - value2 - etc.
getter nodes = [] of Node

# Appends two nodes into this mapping.
def []=(key, value)
@nodes << key << value
end

# Appends a single node into this mapping.
def <<(node)
@nodes << node
end

# Yields each key-value pair in this mapping.
def each
0.step(by: 2, to: @nodes.size - 1) do |i|
yield({@nodes[i], @nodes[i + 1]})
end
end

def to_yaml(builder : YAML::Builder)
builder.mapping(anchor, tag, style) do
each do |key, value|
key.to_yaml(builder)
value.to_yaml(builder)
end
end
end
end

# An alias.
class Alias < Node
# The node this alias points to.
# This is set by `YAML::Nodes.parse`, and is `nil` by default.
property value : Node?

# Creates an alias with tha given *anchor*.
def initialize(@anchor : String)
end

def to_yaml(builder : YAML::Builder)
builder.alias(anchor.not_nil!)
end
end
end
70 changes: 70 additions & 0 deletions src/yaml/nodes/parser.cr
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
# :nodoc:
class YAML::Nodes::Parser < YAML::Parser
def initialize(content : String | IO)
super
@anchors = {} of String => Node
end

def self.new(content)
parser = new(content)
yield parser ensure parser.close
end

def new_documents
[] of Array(Node)
end

def new_document
Document.new
end

def new_sequence
sequence = Sequence.new
set_common_properties(sequence)
sequence.style = @pull_parser.sequence_style
sequence
end

def new_mapping
mapping = Mapping.new
set_common_properties(mapping)
mapping.style = @pull_parser.mapping_style
mapping
end

def new_scalar
scalar = Scalar.new(@pull_parser.value)
set_common_properties(scalar)
scalar.style = @pull_parser.scalar_style
scalar
end

private def set_common_properties(node)
node.tag = @pull_parser.tag
node.anchor = @pull_parser.anchor
node.start_line = @pull_parser.start_line.to_i
node.start_column = @pull_parser.start_column.to_i
end

def end_value(node)
node.end_line = @pull_parser.end_line.to_i
node.end_column = @pull_parser.end_column.to_i
end

def put_anchor(anchor, value)
@anchors[anchor] = value
end

def get_anchor(anchor)
value = @anchors.fetch(anchor) do
@pull_parser.raise("Unknown anchor '#{anchor}'")
end
node = Alias.new(anchor)
node.value = value
set_common_properties(node)
node
end

def process_tag(tag, &block)
end
end
63 changes: 63 additions & 0 deletions src/yaml/parse_context.cr
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
# Parsing context that holds anchors and what they refer to.
#
# When implementing `new(ctx : YAML::ParseContext, node : YAML::Nodes::Node)`
# to deserialize an object from a node, `Reference` types must invoke
# both `read_alias` and `record_anchor` in order to support parsing
# recursive data structures.
#
# - `read_alias` must be invoked before an instance is created
# - `record_anchor` must be invoked after an instance is created and
# before its members are deserialized.
class YAML::ParseContext
def initialize
# Recorded anchors: anchor => {object_id, crystal_type_id}
@anchors = {} of String => {UInt64, Int32}
end

# Associates an object with an anchor.
def record_anchor(node, object : T) : Nil forall T
return unless object.is_a?(Reference)

record_anchor(node.anchor, object.object_id, object.crystal_type_id)
end

private def record_anchor(anchor, object_id, crystal_type_id)
return unless anchor

@anchors[anchor] = {object_id, crystal_type_id}
end

# Tries to read an alias from `node` of type `T`. Invokes
# the block if successful, and invokers must return this object
# instead of deserializing their members.
def read_alias(node, type : T.class) forall T
if ptr = read_alias_impl(node, T.crystal_instance_type_id, raise_on_alias: true)
yield ptr.unsafe_as(T)
end
end

# Similar to `read_alias` but doesn't raise if an alias exists
# but an instance of type T isn't associated with the current anchor.
def read_alias?(node, type : T.class) forall T
if ptr = read_alias_impl(node, T.crystal_instance_type_id, raise_on_alias: false)
yield ptr.unsafe_as(T)
end
end

private def read_alias_impl(node, expected_crystal_type_id, raise_on_alias)
if node.is_a?(YAML::Nodes::Alias)
value = @anchors[node.anchor]?

if value
object_id, crystal_type_id = value
if crystal_type_id == expected_crystal_type_id
return Pointer(Void).new(object_id)
end
end

raise("Error deserailizing alias") if raise_on_alias
end

nil
end
end
190 changes: 128 additions & 62 deletions src/yaml/parser.cr
View file
Original file line number Diff line number Diff line change
@@ -1,109 +1,175 @@
class YAML::Parser
# :nodoc:
abstract class YAML::Parser
def initialize(content : String | IO)
@pull_parser = PullParser.new(content)
@anchors = {} of String => YAML::Type
end

def self.new(content)
parser = new(content)
yield parser ensure parser.close
end

def close
@pull_parser.close
abstract def new_documents
abstract def new_document
abstract def new_sequence
abstract def new_mapping
abstract def new_scalar
abstract def put_anchor(anchor, value)
abstract def get_anchor(anchor)

def end_value(value)
end

def process_tag(tag, &block)
end

protected def cast_value(value)
value
end

protected def cast_document(document)
document
end

# Deserializes multiple YAML document.
def parse_all
documents = [] of YAML::Any
documents = new_documents

@pull_parser.read_next
loop do
case @pull_parser.read_next
when EventKind::STREAM_END
case @pull_parser.kind
when .stream_end?
return documents
when EventKind::DOCUMENT_START
documents << YAML::Any.new(parse_document)
when .document_start?
documents << cast_value(parse_document)
else
unexpected_event
end
end
end

# Deserializes a YAML document.
def parse
value = case @pull_parser.read_next
when EventKind::STREAM_END
nil
when EventKind::DOCUMENT_START
parse_document
else
unexpected_event
end
YAML::Any.new(value)
end

def parse_document
@pull_parser.read_next
value = parse_node
unless @pull_parser.read_next == EventKind::DOCUMENT_END
raise "Expected DOCUMENT_END"

document = new_document

case @pull_parser.kind
when .stream_end?
when .document_start?
parse_document(document)
else
unexpected_event
end
value

cast_value(cast_document(document))
end

private def parse_document
document = new_document
parse_document(document)
cast_document(document)
end

private def parse_document(document)
@pull_parser.read_next
document << parse_node
end_value(document)
@pull_parser.read_document_end
end

def parse_node
protected def parse_node
tag = @pull_parser.tag
if tag
process_tag(tag) do |value|
return value
end
end

case @pull_parser.kind
when EventKind::SCALAR
anchor @pull_parser.value, @pull_parser.scalar_anchor
when EventKind::ALIAS
@anchors[@pull_parser.alias_anchor]
when EventKind::SEQUENCE_START
when .scalar?
parse_scalar
when .alias?
parse_alias
when .sequence_start?
parse_sequence
when EventKind::MAPPING_START
when .mapping_start?
parse_mapping
else
unexpected_event
end
end

def parse_sequence
sequence = [] of YAML::Type
anchor sequence, @pull_parser.sequence_anchor
protected def parse_scalar
value = anchor(@pull_parser.anchor, new_scalar)
@pull_parser.read_next
value
end

loop do
case @pull_parser.read_next
when EventKind::SEQUENCE_END
return sequence
else
sequence << parse_node
end
protected def parse_alias
value = get_anchor(@pull_parser.anchor.not_nil!)
@pull_parser.read_next
value
end

protected def parse_sequence
sequence = anchor new_sequence

parse_sequence(sequence) do
sequence << parse_node
end

sequence
end

def parse_mapping
mapping = {} of YAML::Type => YAML::Type
anchor mapping, @pull_parser.mapping_anchor
protected def parse_sequence(sequence)
@pull_parser.read_sequence_start

loop do
case @pull_parser.read_next
when EventKind::MAPPING_END
return mapping
else
key = parse_node
tag = @pull_parser.tag
@pull_parser.read_next
value = parse_node
if key == "<<" && value.is_a?(Hash) && tag != "tag:yaml.org,2002:str"
mapping.merge!(value)
else
mapping[key] = value
end
end
until @pull_parser.kind.sequence_end?
yield
end

end_value(sequence)

@pull_parser.read_next
end

def anchor(value, anchor)
@anchors[anchor] = value if anchor
protected def parse_mapping
mapping = anchor new_mapping

parse_mapping(mapping) do
mapping[parse_node] = parse_node
end

mapping
end

protected def parse_mapping(mapping)
@pull_parser.read_mapping_start

until @pull_parser.kind.mapping_end?
yield
end

end_value(mapping)

@pull_parser.read_next
end

# Closes this parser, freeing up resources.
def close
@pull_parser.close
end

private def anchor(anchor, value)
put_anchor(anchor, value) if anchor
value
end

private def anchor(value)
anchor(@pull_parser.anchor, value)
end

private def unexpected_event
raise "Unexpected event: #{@pull_parser.kind}"
end
195 changes: 102 additions & 93 deletions src/yaml/pull_parser.cr
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,12 @@
# A pull parser allows parsing a YAML document by events.
#
# When creating an instance, the parser is positioned in
# the first event. To get the event kind invoke `kind`.
# If the event is a scalar you can invoke `value` to get
# its **string** value. Other methods like `tag`, `anchor`
# and `scalar_style` let you inspect other information from events.
#
# Invoking `read_next` reads the next event.
class YAML::PullParser
protected getter content

@@ -22,58 +31,84 @@ class YAML::PullParser
end

read_next
raise "Expected STREAM_START" unless kind == LibYAML::EventType::STREAM_START
raise "Expected STREAM_START" unless kind.stream_start?
end

# Creates a parser, yields it to the block, and closes
# the parser at the end of it.
def self.new(content)
parser = new(content)
yield parser ensure parser.close
end

def kind
# The current event kind.
def kind : EventKind
@event.type
end

def tag
ptr = @event.data.scalar.tag
# Returns the tag associated to the current event, or `nil`
# if there's no tag.
def tag : String?
case kind
when .mapping_start?
ptr = @event.data.mapping_start.tag
when .sequence_start?
ptr = @event.data.sequence_start.tag
when .scalar?
ptr = @event.data.scalar.tag
end
ptr ? String.new(ptr) : nil
end

def value
# Returns the scalar value, assuming the pull parser
# is located at a scalar. Raises otherwise.
def value : String
expect_kind EventKind::SCALAR

ptr = @event.data.scalar.value
ptr ? String.new(ptr, @event.data.scalar.length) : nil
ptr ? String.new(ptr, @event.data.scalar.length) : ""
end

# Returns the anchor associated to the current event, or `nil`
# if there's no anchor.
def anchor
case kind
when LibYAML::EventType::SCALAR
scalar_anchor
when LibYAML::EventType::SEQUENCE_START
sequence_anchor
when LibYAML::EventType::MAPPING_START
mapping_anchor
when .scalar?
read_anchor @event.data.scalar.anchor
when .sequence_start?
read_anchor @event.data.sequence_start.anchor
when .mapping_start?
read_anchor @event.data.mapping_start.anchor
when .alias?
read_anchor @event.data.alias.anchor
else
nil
end
end

def scalar_anchor
read_anchor @event.data.scalar.anchor
# Returns the sequence style, assuming the pull parser is located
# at a sequence begin event. Raises otherwise.
def sequence_style : SequenceStyle
expect_kind EventKind::SEQUENCE_START
@event.data.sequence_start.style
end

def sequence_anchor
read_anchor @event.data.sequence_start.anchor
# Returns the mapping style, assuming the pull parser is located
# at a mapping begin event. Raises otherwise.
def mapping_style : MappingStyle
expect_kind EventKind::MAPPING_START
@event.data.mapping_start.style
end

def mapping_anchor
read_anchor @event.data.mapping_start.anchor
end

def alias_anchor
read_anchor @event.data.alias.anchor
# Returns the scalar style, assuming the pull parser is located
# at a scalar event. Raises otherwise.
def scalar_style : ScalarStyle
expect_kind EventKind::SCALAR
@event.data.scalar.style
end

def read_next
# Reads the next event.
def read_next : EventKind
LibYAML.yaml_event_delete(pointerof(@event))
LibYAML.yaml_parser_parse(@parser, pointerof(@event))
if problem = problem?
@@ -87,154 +122,119 @@ class YAML::PullParser
kind
end

# Reads a "stream start" event, yields to the block,
# and then reads a "stream end" event.
def read_stream
read_stream_start
value = yield
read_stream_end
value
end

# Reads a "document start" event, yields to the block,
# and then reads a "document end" event.
def read_document
read_document_start
value = yield
read_document_end
value
end

# Reads a "sequence start" event, yields to the block,
# and then reads a "sequence end" event.
def read_sequence
read_sequence_start
value = yield
read_sequence_end
value
end

# Reads a "mapping start" event, yields to the block,
# and then reads a "mapping end" event.
def read_mapping
read_mapping_start
value = yield
read_mapping_end
value
end

# Reads an alias event, returning its anchor.
def read_alias
expect_kind EventKind::ALIAS
anchor = alias_anchor
anchor = self.anchor
read_next
anchor
end

# Reads a scalar, returning its value.
def read_scalar
expect_kind EventKind::SCALAR
value = self.value.not_nil!
value = self.value
read_next
value
end

# Reads a "stream start" event.
def read_stream_start
read EventKind::STREAM_START
end

# Reads a "stream end" event.
def read_stream_end
read EventKind::STREAM_END
end

# Reads a "document start" event.
def read_document_start
read EventKind::DOCUMENT_START
end

# Reads a "document end" event.
def read_document_end
read EventKind::DOCUMENT_END
end

# Reads a "sequence start" event.
def read_sequence_start
read EventKind::SEQUENCE_START
end

# Reads a "sequence end" event.
def read_sequence_end
read EventKind::SEQUENCE_END
end

# Reads a "mapping start" event.
def read_mapping_start
read EventKind::MAPPING_START
end

# Reads a "mapping end" event.
def read_mapping_end
read EventKind::MAPPING_END
end

def read_null_or
if kind == EventKind::SCALAR && (value = self.value).nil? || (value && value.empty?)
read_next
nil
else
yield
end
end

def read(expected_kind)
# Reads an expected event kind.
def read(expected_kind : EventKind) : EventKind
expect_kind expected_kind
read_next
end

def read_raw
case kind
when EventKind::SCALAR
self.value.not_nil!.tap { read_next }
when EventKind::SEQUENCE_START, EventKind::MAPPING_START
String.build { |io| read_raw(io) }
else
raise "Unexpected kind: #{kind}"
end
end

def read_raw(io)
case kind
when EventKind::SCALAR
self.value.not_nil!.inspect(io)
read_next
when EventKind::SEQUENCE_START
io << "["
read_next
first = true
while kind != EventKind::SEQUENCE_END
io << "," unless first
read_raw(io)
first = false
end
io << "]"
read_next
when EventKind::MAPPING_START
io << "{"
read_next
first = true
while kind != EventKind::MAPPING_END
io << "," unless first
read_raw(io)
io << ":"
read_raw(io)
first = false
end
io << "}"
read_next
else
raise "Unexpected kind: #{kind}"
end
end

def skip
case kind
when EventKind::SCALAR
when .scalar?
read_next
when EventKind::ALIAS
when .alias?
read_next
when EventKind::SEQUENCE_START
when .sequence_start?
read_next
while kind != EventKind::SEQUENCE_END
until kind.sequence_end?
skip
end
read_next
when EventKind::MAPPING_START
when .mapping_start?
read_next
while kind != EventKind::MAPPING_END
until kind.mapping_end?
skip
skip
end
@@ -245,23 +245,31 @@ class YAML::PullParser
# Note: YAML starts counting from 0, we want to count from 1

def location
{line_number, column_number}
{start_line, start_column}
end

def line_number
def start_line
@event.start_mark.line + 1
end

def column_number
def start_column
@event.start_mark.column + 1
end

def end_line
@event.end_mark.line + 1
end

def end_column
@event.end_mark.column + 1
end

private def problem_line_number
(problem? ? problem_mark?.line : line_number) + 1
(problem? ? problem_mark?.line : start_line) + 1
end

private def problem_column_number
(problem? ? problem_mark?.column : column_number) + 1
(problem? ? problem_mark?.column : start_column) + 1
end

private def problem_mark?
@@ -302,15 +310,16 @@ class YAML::PullParser
@closed = true
end

private def expect_kind(kind)
# Raises if the current kind is not the expected one.
def expect_kind(kind : EventKind)
raise "Expected #{kind} but was #{self.kind}" unless kind == self.kind
end

private def read_anchor(anchor)
anchor ? String.new(anchor) : nil
end

def raise(msg : String, line_number = self.line_number, column_number = self.column_number, context_info = nil)
def raise(msg : String, line_number = self.start_line, column_number = self.start_column, context_info = nil)
::raise ParseException.new(msg, line_number, column_number, context_info)
end
end
335 changes: 335 additions & 0 deletions src/yaml/schema/core.cr
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,335 @@
# Provides utility methods for the YAML 1.1 core schema
# with the additional independent types specified in http://yaml.org/type/
module YAML::Schema::Core
# Deserializes a YAML document.
#
# Same as `YAML.parse`.
def self.parse(data : String | IO)
Parser.new data, &.parse
end

# Deserializes multiple YAML documents.
#
# Same as `YAML.parse_all`.
def self.parse_all(data : String | IO)
Parser.new data, &.parse_all
end

# Assuming the *pull_parser* is positioned in a scalar,
# parses it according to the core schema, taking the
# scalar's style and tag into account, then advances
# the pull parser.
def self.parse_scalar(pull_parser : YAML::PullParser) : Type
string = pull_parser.value

# Check for core schema tags
process_scalar_tag(pull_parser, pull_parser.tag) do |value|
return value
end

# Non-plain scalar is always a string
unless pull_parser.scalar_style.plain?
return string
end

parse_scalar(string)
end

# Parses a scalar value from the given *node*.
def self.parse_scalar(node : YAML::Nodes::Scalar) : Type
string = node.value

# Check for core schema tags
process_scalar_tag(node) do |value|
return value
end

# Non-plain scalar is always a string
unless node.style.plain?
return string
end

parse_scalar(string)
end

# Parses a string according to the core schema, assuming
# the string had a plain style.
#
# ```
# YAML::Schema::Core.parse_scalar("hello") # => "hello"
# YAML::Schema::Core.parse_scalar("1.2") # => 1.2
# YAML::Schema::Core.parse_scalar("false") # => false
# ```
def self.parse_scalar(string : String) : Type
if parse_null?(string)
return nil
end

value = parse_bool?(string)
return value unless value.nil?

value = parse_float_infinity_and_nan?(string)
return value if value

# Optimizations for prefixes that either parse to
# a number or are strings otherwise
case string
when .starts_with?("0x"),
.starts_with?("+0x"),
.starts_with?("-0x")
value = string.to_i64?(base: 16, prefix: true)
return value || string
when .starts_with?('0')
value = string.to_i64?(base: 8, prefix: true)
return value || string
when .starts_with?('-'),
.starts_with?('+')
value = parse_number?(string)
return value || string
end

if string[0].ascii_number?
value = parse_number?(string)
return value if value

value = parse_time?(string)
return value if value
end

string
end

# Returns whether a string is reserved and must non be output
# with a plain style, according to the core schema.
#
# ```
# YAML::Schema::Core.reserved_string?("hello") # => false
# YAML::Schema::Core.reserved_string?("1.2") # => true
# YAML::Schema::Core.reserved_string?("false") # => true
# ```
def self.reserved_string?(string) : Bool
# There's simply no other way than parsing the string and
# checking what we got.
#
# The performance loss is minimal because `parse_scalar`
# doesn't allocate memory: it can only return primitive
# types, or `Time`, which is a struct.
!parse_scalar(string).is_a?(String)
end

# If `node` parses to a null value, returns `nil`, otherwise
# invokes the given block.
def self.parse_null_or(node : YAML::Nodes::Node)
if node.is_a?(YAML::Nodes::Scalar) && parse_null?(node.value)
nil
else
yield
end
end

# Invokes the block for each of the given *node*s keys and
# values, resolving merge keys (<<) when found (keys and
# values of the resolved merge mappings are yielded,
# recursively).
def self.each(node : YAML::Nodes::Mapping)
# We can't just traverse the nodes and invoke yield because
# yield can't recurse. So, we use a stack of {Mapping, index}.
# We pop from the stack and traverse the mapping values.
# When we find a merge, we stop (put back in the stack with
# that mapping and next index) and add solved mappings from
# the merge to the stack, and continue processing.

stack = [{node, 0}]

# Mappings that we already visited. In case of a recursion
# we want to stop. For example:
#
# foo: &foo
# <<: *foo
#
# When we traverse &foo we'll put it in visited,
# and when we find it in *foo we'll skip it.
#
# This has no use case, but we don't want to hang the program.
visited = Set(YAML::Nodes::Mapping).new

until stack.empty?
mapping, index = stack.pop

visited << mapping

while index < mapping.nodes.size
key = mapping.nodes[index]
index += 1

value = mapping.nodes[index]
index += 1

if key.is_a?(YAML::Nodes::Scalar) &&
key.value == "<<" &&
key.tag != "tag:yaml.org,2002:str" &&
solve_merge(stack, mapping, index, value, visited)
break
else
yield({key, value})
end
end
end
end

private def self.solve_merge(stack, mapping, index, value, visited)
value = value.value if value.is_a?(YAML::Nodes::Alias)

case value
when YAML::Nodes::Mapping
stack.push({mapping, index})

unless visited.includes?(value)
stack.push({value, 0})
end

true
when YAML::Nodes::Sequence
all_mappings = value.nodes.all? do |elem|
elem = elem.value if elem.is_a?(YAML::Nodes::Alias)
elem.is_a?(YAML::Nodes::Mapping)
end

if all_mappings
stack.push({mapping, index})

value.each do |elem|
elem = elem.value if elem.is_a?(YAML::Nodes::Alias)
mapping = elem.as(YAML::Nodes::Mapping)

unless visited.includes?(mapping)
stack.push({mapping, 0})
end
end

true
else
false
end
else
false
end
end

protected def self.parse_binary(string, location) : Bytes
Base64.decode(string)
rescue ex : Base64::Error
raise YAML::ParseException.new("Error decoding Base64: #{ex.message}", *location)
end

protected def self.parse_bool(string, location) : Bool
value = parse_bool?(string)
unless value.nil?
return value
end

raise YAML::ParseException.new("Invalid bool", *location)
end

protected def self.parse_int(string, location) : Int64
string.to_i64?(underscore: true, prefix: true) ||
raise(YAML::ParseException.new("Invalid int", *location))
end

protected def self.parse_float(string, location) : Float64
parse_float_infinity_and_nan?(string) ||
parse_float?(string) ||
raise(YAML::ParseException.new("Invalid float", *location))
end

protected def self.parse_null(string, location) : Nil
if parse_null?(string)
return nil
end

raise YAML::ParseException.new("Invalid null", *location)
end

protected def self.parse_time(string, location) : Time
parse_time?(string) ||
raise(YAML::ParseException.new("Invalid timestamp", *location))
end

protected def self.process_scalar_tag(scalar)
process_scalar_tag(scalar, scalar.tag) do |value|
yield value
end
end

protected def self.process_scalar_tag(source, tag)
case tag
when "tag:yaml.org,2002:binary"
yield parse_binary(source.value, source.location)
when "tag:yaml.org,2002:bool"
yield parse_bool(source.value, source.location)
when "tag:yaml.org,2002:float"
yield parse_float(source.value, source.location)
when "tag:yaml.org,2002:int"
yield parse_int(source.value, source.location)
when "tag:yaml.org,2002:null"
yield parse_null(source.value, source.location)
when "tag:yaml.org,2002:str"
yield source.value
when "tag:yaml.org,2002:timestamp"
yield parse_time(source.value, source.location)
end
end

private def self.parse_null?(string)
case string
when .empty?, "~", "null", "Null", "NULL"
true
else
false
end
end

private def self.parse_bool?(string)
case string
when "yes", "Yes", "YES", "true", "True", "TRUE", "on", "On", "ON"
true
when "no", "No", "NO", "false", "False", "FALSE", "off", "Off", "OFF"
false
else
nil
end
end

private def self.parse_number?(string)
parse_int?(string) || parse_float?(string)
end

private def self.parse_int?(string)
string.to_i64?(underscore: true)
end

private def self.parse_float?(string)
string = string.delete('_') if string.includes?('_')
string.to_f64?
end

private def self.parse_float_infinity_and_nan?(string)
case string
when ".inf", ".Inf", ".INF", "+.inf", "+.Inf", "+.INF"
Float64::INFINITY
when "-.inf", "-.Inf", "-.INF"
-Float64::INFINITY
when ".nan", ".NaN", ".NAN"
Float64::NAN
else
nil
end
end

private def self.parse_time?(string)
# Minimum length is that of YYYY-M-D
return nil if string.size < 8

TimeParser.new(string).parse
end
end
Loading

0 comments on commit 3335450

Please sign in to comment.