From b613a2c5ae03c425bc94771e4eced1b80e4e80d0 Mon Sep 17 00:00:00 2001 From: hsbt Date: Mon, 27 Nov 2017 03:11:18 +0000 Subject: [PATCH] Merge psych-3.0.0.beta4 from upstream. git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@60917 b2dd03c8-39d4-4d8f-98ff-823fe69b080e --- ext/psych/lib/psych.rb | 28 +++++++--- ext/psych/lib/psych/handler.rb | 5 ++ ext/psych/lib/psych/nodes/node.rb | 12 +++++ ext/psych/lib/psych/tree_builder.rb | 48 ++++++++++++++++-- ext/psych/lib/psych/versions.rb | 2 +- ext/psych/lib/psych/visitors/yaml_tree.rb | 2 +- ext/psych/psych.gemspec | 4 +- ext/psych/psych_parser.c | 49 +++++++++++++----- test/psych/test_encoding.rb | 12 +++++ test/psych/test_parser.rb | 62 +++++++++++++++++++++-- test/psych/test_psych.rb | 18 +++++++ test/psych/test_string.rb | 7 +++ test/psych/test_tree_builder.rb | 14 +++++ 13 files changed, 231 insertions(+), 32 deletions(-) diff --git a/ext/psych/lib/psych.rb b/ext/psych/lib/psych.rb index e93ac8f406..dfb6c1b00e 100644 --- a/ext/psych/lib/psych.rb +++ b/ext/psych/lib/psych.rb @@ -198,12 +198,13 @@ require 'psych/class_loader' # # ==== Receiving an events stream # -# parser = Psych::Parser.new(Psych::Handlers::Recorder.new) +# recorder = Psych::Handlers::Recorder.new +# parser = Psych::Parser.new(recorder) # # parser.parse("---\n - a\n - b") -# parser.events # => [list of [event, args] lists] -# # event is one of: Psych::Handler::EVENTS -# # args are the arguments passed to the event +# recorder.events # => [list of [event, args] lists] +# # event is one of: Psych::Handler::EVENTS +# # args are the arguments passed to the event # # === Emitting # @@ -251,9 +252,11 @@ module Psych # ex.file # => 'file.txt' # ex.message # => "(file.txt): found character that cannot start any token" # end - def self.load yaml, filename = nil, fallback = false + def self.load yaml, filename = nil, fallback = false, symbolize_names: false result = parse(yaml, filename, fallback) - result ? result.to_ruby : result + result = result.to_ruby if result + symbolize_names!(result) if symbolize_names + result end ### @@ -502,6 +505,19 @@ module Psych @dump_tags[klass] = tag end + def self.symbolize_names!(result) + case result + when Hash + result.keys.each do |key| + result[key.to_sym] = symbolize_names!(result.delete(key)) + end + when Array + result.map! { |r| symbolize_names!(r) } + end + result + end + private_class_method :symbolize_names! + class << self attr_accessor :load_tags attr_accessor :dump_tags diff --git a/ext/psych/lib/psych/handler.rb b/ext/psych/lib/psych/handler.rb index 1074c18f9e..84a3b4f2bc 100644 --- a/ext/psych/lib/psych/handler.rb +++ b/ext/psych/lib/psych/handler.rb @@ -241,6 +241,11 @@ module Psych def end_stream end + ### + # Called before each event with line/column information. + def event_location(start_line, start_column, end_line, end_column) + end + ### # Is this handler a streaming handler? def streaming? diff --git a/ext/psych/lib/psych/nodes/node.rb b/ext/psych/lib/psych/nodes/node.rb index 1c7672164d..6d86669a17 100644 --- a/ext/psych/lib/psych/nodes/node.rb +++ b/ext/psych/lib/psych/nodes/node.rb @@ -17,6 +17,18 @@ module Psych # An associated tag attr_reader :tag + # The line number where this node start + attr_accessor :start_line + + # The column number where this node start + attr_accessor :start_column + + # The line number where this node ends + attr_accessor :end_line + + # The column number where this node ends + attr_accessor :end_column + # Create a new Psych::Nodes::Node def initialize @children = [] diff --git a/ext/psych/lib/psych/tree_builder.rb b/ext/psych/lib/psych/tree_builder.rb index b10fd5c5cd..47a1695643 100644 --- a/ext/psych/lib/psych/tree_builder.rb +++ b/ext/psych/lib/psych/tree_builder.rb @@ -23,6 +23,18 @@ module Psych @stack = [] @last = nil @root = nil + + @start_line = nil + @start_column = nil + @end_line = nil + @end_column = nil + end + + def event_location(start_line, start_column, end_line, end_column) + @start_line = start_line + @start_column = start_column + @end_line = end_line + @end_column = end_column end %w{ @@ -32,12 +44,15 @@ module Psych class_eval %{ def start_#{node.downcase}(anchor, tag, implicit, style) n = Nodes::#{node}.new(anchor, tag, implicit, style) + set_start_location(n) @last.children << n push n end def end_#{node.downcase} - pop + n = pop + set_end_location(n) + n end } end @@ -49,6 +64,7 @@ module Psych # See Psych::Handler#start_document def start_document version, tag_directives, implicit n = Nodes::Document.new version, tag_directives, implicit + set_start_location(n) @last.children << n push n end @@ -60,26 +76,35 @@ module Psych # See Psych::Handler#start_document def end_document implicit_end = !streaming? @last.implicit_end = implicit_end - pop + n = pop + set_end_location(n) + n end def start_stream encoding @root = Nodes::Stream.new(encoding) + set_start_location(@root) push @root end def end_stream - pop + n = pop + set_end_location(n) + n end def scalar value, anchor, tag, plain, quoted, style s = Nodes::Scalar.new(value,anchor,tag,plain,quoted,style) + set_location(s) @last.children << s s end def alias anchor - @last.children << Nodes::Alias.new(anchor) + a = Nodes::Alias.new(anchor) + set_location(a) + @last.children << a + a end private @@ -93,5 +118,20 @@ module Psych @last = @stack.last x end + + def set_location(node) + set_start_location(node) + set_end_location(node) + end + + def set_start_location(node) + node.start_line = @start_line + node.start_column = @start_column + end + + def set_end_location(node) + node.end_line = @end_line + node.end_column = @end_column + end end end diff --git a/ext/psych/lib/psych/versions.rb b/ext/psych/lib/psych/versions.rb index 34a2c2607b..cf5fc4116e 100644 --- a/ext/psych/lib/psych/versions.rb +++ b/ext/psych/lib/psych/versions.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Psych # The version is Psych you're using - VERSION = '3.0.0.beta3' + VERSION = '3.0.0.beta4' if RUBY_ENGINE == 'jruby' DEFAULT_SNAKEYAML_VERSION = '1.18'.freeze diff --git a/ext/psych/lib/psych/visitors/yaml_tree.rb b/ext/psych/lib/psych/visitors/yaml_tree.rb index cfed8f1814..f44e973c52 100644 --- a/ext/psych/lib/psych/visitors/yaml_tree.rb +++ b/ext/psych/lib/psych/visitors/yaml_tree.rb @@ -304,7 +304,7 @@ module Psych quote = false elsif @line_width && o.length > @line_width style = Nodes::Scalar::FOLDED - elsif o =~ /^[^[:word:]][^"]*$/ + elsif o =~ /^[^[:word:]][^"]*$/ or o =~ /^([^"]*'+[^"]*)+$/ style = Nodes::Scalar::DOUBLE_QUOTED elsif not String === @ss.tokenize(o) or /\A0[0-7]*[89]/ =~ o style = Nodes::Scalar::SINGLE_QUOTED diff --git a/ext/psych/psych.gemspec b/ext/psych/psych.gemspec index 054f209893..bbebce8776 100644 --- a/ext/psych/psych.gemspec +++ b/ext/psych/psych.gemspec @@ -3,10 +3,10 @@ Gem::Specification.new do |s| s.name = "psych" - s.version = "3.0.0.beta3" + s.version = "3.0.0.beta4" s.authors = ["Aaron Patterson", "SHIBATA Hiroshi", "Charles Oliver Nutter"] s.email = ["aaron@tenderlovemaking.com", "hsbt@ruby-lang.org", "headius@headius.com"] - s.date = "2017-06-16" + s.date = "2017-11-27" s.summary = "Psych is a YAML parser and emitter" s.description = <<-DESCRIPTION Psych is a YAML parser and emitter. Psych leverages libyaml[http://pyyaml.org/wiki/LibYAML] diff --git a/ext/psych/psych_parser.c b/ext/psych/psych_parser.c index 47ed8744b6..4b18c94761 100644 --- a/ext/psych/psych_parser.c +++ b/ext/psych/psych_parser.c @@ -16,6 +16,7 @@ static ID id_start_sequence; static ID id_end_sequence; static ID id_start_mapping; static ID id_end_mapping; +static ID id_event_location; #define PSYCH_TRANSCODE(_str, _yaml_enc, _internal_enc) \ do { \ @@ -232,6 +233,12 @@ static VALUE protected_end_stream(VALUE handler) return rb_funcall(handler, id_end_stream, 0); } +static VALUE protected_event_location(VALUE pointer) +{ + VALUE *args = (VALUE *)pointer; + return rb_funcall3(args[0], id_event_location, 4, args + 1); +} + /* * call-seq: * parser.parse(yaml) @@ -295,6 +302,21 @@ static VALUE parse(int argc, VALUE *argv, VALUE self) rb_exc_raise(exception); } + VALUE event_args[5]; + VALUE start_line, start_column, end_line, end_column; + + start_line = INT2NUM((long)event.start_mark.line); + start_column = INT2NUM((long)event.start_mark.column); + end_line = INT2NUM((long)event.end_mark.line); + end_column = INT2NUM((long)event.end_mark.column); + + event_args[0] = handler; + event_args[1] = start_line; + event_args[2] = start_column; + event_args[3] = end_line; + event_args[4] = end_column; + rb_protect(protected_event_location, (VALUE)event_args, &state); + switch(event.type) { case YAML_STREAM_START_EVENT: { @@ -551,18 +573,19 @@ void Init_psych_parser(void) rb_define_method(cPsychParser, "parse", parse, -1); rb_define_method(cPsychParser, "mark", mark, 0); - id_read = rb_intern("read"); - id_path = rb_intern("path"); - id_empty = rb_intern("empty"); - id_start_stream = rb_intern("start_stream"); - id_end_stream = rb_intern("end_stream"); - id_start_document = rb_intern("start_document"); - id_end_document = rb_intern("end_document"); - id_alias = rb_intern("alias"); - id_scalar = rb_intern("scalar"); - id_start_sequence = rb_intern("start_sequence"); - id_end_sequence = rb_intern("end_sequence"); - id_start_mapping = rb_intern("start_mapping"); - id_end_mapping = rb_intern("end_mapping"); + id_read = rb_intern("read"); + id_path = rb_intern("path"); + id_empty = rb_intern("empty"); + id_start_stream = rb_intern("start_stream"); + id_end_stream = rb_intern("end_stream"); + id_start_document = rb_intern("start_document"); + id_end_document = rb_intern("end_document"); + id_alias = rb_intern("alias"); + id_scalar = rb_intern("scalar"); + id_start_sequence = rb_intern("start_sequence"); + id_end_sequence = rb_intern("end_sequence"); + id_start_mapping = rb_intern("start_mapping"); + id_end_mapping = rb_intern("end_mapping"); + id_event_location = rb_intern("event_location"); } /* vim: set noet sws=4 sw=4: */ diff --git a/test/psych/test_encoding.rb b/test/psych/test_encoding.rb index a4f9f036fd..ef6653142f 100644 --- a/test/psych/test_encoding.rb +++ b/test/psych/test_encoding.rb @@ -106,6 +106,18 @@ module Psych } end + def test_io_utf8_read_as_binary + Tempfile.create(['utf8', 'yml']) {|t| + t.binmode + t.write '--- こんにちは！'.encode('UTF-8') + t.close + + File.open(t.path, 'rb', :encoding => 'ascii-8bit') do |f| + assert_equal "こんにちは！", Psych.load(f) + end + } + end + def test_emit_alias @emitter.start_stream Psych::Parser::UTF8 @emitter.start_document [], [], true diff --git a/test/psych/test_parser.rb b/test/psych/test_parser.rb index 6b554cedf0..e8225dabb6 100644 --- a/test/psych/test_parser.rb +++ b/test/psych/test_parser.rb @@ -87,13 +87,22 @@ module Psych assert_equal 0, @parser.mark.line @parser.parse "---\n- hello\n- world" line_calls = @handler.marks.map(&:line).zip(@handler.calls.map(&:first)) - assert_equal [[0, :start_stream], + assert_equal [ + [0, :event_location], + [0, :start_stream], + [0, :event_location], [0, :start_document], + [1, :event_location], [1, :start_sequence], + [2, :event_location], [2, :scalar], + [3, :event_location], [3, :scalar], + [3, :event_location], [3, :end_sequence], + [3, :event_location], [3, :end_document], + [3, :event_location], [3, :end_stream]], line_calls assert_equal 3, @parser.mark.line @@ -103,13 +112,22 @@ module Psych assert_equal 0, @parser.mark.column @parser.parse "---\n- hello\n- world" col_calls = @handler.marks.map(&:column).zip(@handler.calls.map(&:first)) - assert_equal [[0, :start_stream], + assert_equal [ + [0, :event_location], + [0, :start_stream], + [3, :event_location], [3, :start_document], + [1, :event_location], [1, :start_sequence], + [0, :event_location], [0, :scalar], + [0, :event_location], [0, :scalar], + [0, :event_location], [0, :end_sequence], + [0, :event_location], [0, :end_document], + [0, :event_location], [0, :end_stream]], col_calls assert_equal 0, @parser.mark.column @@ -119,13 +137,22 @@ module Psych assert_equal 0, @parser.mark.index @parser.parse "---\n- hello\n- world" idx_calls = @handler.marks.map(&:index).zip(@handler.calls.map(&:first)) - assert_equal [[0, :start_stream], + assert_equal [ + [0, :event_location], + [0, :start_stream], + [3, :event_location], [3, :start_document], + [5, :event_location], [5, :start_sequence], + [12, :event_location], [12, :scalar], + [19, :event_location], [19, :scalar], + [19, :event_location], [19, :end_sequence], + [19, :event_location], [19, :end_document], + [19, :event_location], [19, :end_stream]], idx_calls assert_equal 19, @parser.mark.index @@ -137,7 +164,7 @@ module Psych # BOM + text yml = "\uFEFF#{tadpole}".encode('UTF-16LE') @parser.parse yml - assert_equal tadpole, @parser.handler.calls[2][1].first + assert_equal tadpole, @parser.handler.calls.find { |method, args| method == :scalar }[1].first end def test_external_encoding @@ -145,7 +172,7 @@ module Psych @parser.external_encoding = Psych::Parser::UTF16LE @parser.parse tadpole.encode 'UTF-16LE' - assert_equal tadpole, @parser.handler.calls[2][1].first + assert_equal tadpole, @parser.handler.calls.find { |method, args| method == :scalar }[1].first end def test_bogus_io @@ -324,6 +351,31 @@ module Psych assert_called :start_document, [[], [['!yaml!', 'tag:yaml.org,2002']], false] end + def test_event_location + @parser.parse "foo:\n" \ + " barbaz: [1, 2]" + + events = @handler.calls.each_slice(2).map do |location, event| + [event[0], location[1]] + end + + assert_equal [ + [:start_stream, [0, 0, 0, 0]], + [:start_document, [0, 0, 0, 0]], + [:start_mapping, [0, 0, 0, 0]], + [:scalar, [0, 0, 0, 3]], + [:start_mapping, [1, 2, 1, 2]], + [:scalar, [1, 2, 1, 8]], + [:start_sequence, [1, 10, 1, 11]], + [:scalar, [1, 11, 1, 12]], + [:scalar, [1, 14, 1, 15]], + [:end_sequence, [1, 15, 1, 16]], + [:end_mapping, [2, 0, 2, 0]], + [:end_mapping, [2, 0, 2, 0]], + [:end_document, [2, 0, 2, 0]], + [:end_stream, [2, 0, 2, 0]]], events + end + def assert_called call, with = nil, parser = @parser if with call = parser.handler.calls.find { |x| diff --git a/test/psych/test_psych.rb b/test/psych/test_psych.rb index d0de95627e..ff5ffbafc6 100644 --- a/test/psych/test_psych.rb +++ b/test/psych/test_psych.rb @@ -182,4 +182,22 @@ class TestPsych < Psych::TestCase ["tag:example.com,2002:foo", "bar"] ], types end + + def test_symbolize_names + result = Psych.load(<<-eoyml) +foo: + bar: baz +hoge: + - fuga: piyo + eoyml + assert_equal result, { "foo" => { "bar" => "baz"}, "hoge" => [{ "fuga" => "piyo" }] } + + result = Psych.load(<<-eoyml, symbolize_names: true) +foo: + bar: baz +hoge: + - fuga: piyo + eoyml + assert_equal result, { foo: { bar: "baz" }, hoge: [{ fuga: "piyo" }] } + end end diff --git a/test/psych/test_string.rb b/test/psych/test_string.rb index 973f38b9c2..e80d19194b 100644 --- a/test/psych/test_string.rb +++ b/test/psych/test_string.rb @@ -37,6 +37,13 @@ module Psych assert_equal str, Psych.load(yaml) end + def test_doublequotes_when_there_are_single_quotes_only + str = "psych: Please don't escape ' with ' here." + yaml = Psych.dump str + assert_equal "--- \"psych: Please don't escape ' with ' here.\"\n", yaml + assert_equal str, Psych.load(yaml) + end + def test_plain_when_shorten_than_line_width_and_no_final_line_break str = "Lorem ipsum" yaml = Psych.dump str, line_width: 12 diff --git a/test/psych/test_tree_builder.rb b/test/psych/test_tree_builder.rb index 3e35788e35..dfb5da9892 100644 --- a/test/psych/test_tree_builder.rb +++ b/test/psych/test_tree_builder.rb @@ -21,6 +21,7 @@ module Psych def test_stream assert_instance_of Nodes::Stream, @tree + assert_location 0, 0, 8, 0, @tree end def test_documents @@ -31,6 +32,7 @@ module Psych assert_equal [1,1], doc.version assert_equal [], doc.tag_directives assert_equal false, doc.implicit + assert_location 0, 0, 8, 0, doc end def test_sequence @@ -43,6 +45,7 @@ module Psych assert_nil seq.tag assert_equal true, seq.implicit assert_equal Nodes::Sequence::BLOCK, seq.style + assert_location 2, 0, 8, 0, seq end def test_scalar @@ -58,6 +61,7 @@ module Psych assert_equal true, scalar.plain assert_equal false, scalar.quoted assert_equal Nodes::Scalar::PLAIN, scalar.style + assert_location 2, 2, 2, 5, scalar end def test_mapping @@ -66,6 +70,7 @@ module Psych map = seq.children[1] assert_instance_of Nodes::Mapping, map + assert_location 3, 2, 6, 1, map end def test_alias @@ -75,6 +80,15 @@ module Psych al = seq.children[2] assert_instance_of Nodes::Alias, al assert_equal 'A', al.anchor + assert_location 7, 2, 7, 4, al + end + + private + def assert_location(start_line, start_column, end_line, end_column, node) + assert_equal start_line, node.start_line + assert_equal start_column, node.start_column + assert_equal end_line, node.end_line + assert_equal end_column, node.end_column end end end