ruby--ruby/test/rdoc/test_rdoc_markup_parser.rb

# coding: utf-8

require 'rdoc/test_case'

class TestRDocMarkupParser < RDoc::TestCase

  def setup
    super

    @have_byteslice = ''.respond_to? :byteslice

    @RMP = @RM::Parser
  end

  def test_build_heading
    parser = @RMP.new

    parser.tokens.replace [
      [:TEXT,    'heading three',  4, 0],
      [:NEWLINE, "\n",            17, 0],
    ]

    assert_equal @RM::Heading.new(3, 'heading three'), parser.build_heading(3)
  end

  def test_char_pos
    parser = @RMP.new
    s = parser.setup_scanner 'cät'

    s.scan(/\S+/)

    if @have_byteslice or @have_encoding then
      assert_equal 3, parser.char_pos(s.pos)
    else
      assert_equal 4, parser.char_pos(s.pos)
    end
  end

  def test_get
    parser = util_parser

    assert_equal [:HEADER, 1, 0, 0], parser.get

    assert_equal 7, parser.tokens.length
  end

  def test_parse_bullet
    str = <<-STR
* l1
* l2
    STR

    expected = [
      @RM::List.new(:BULLET, *[
        @RM::ListItem.new(nil,
          @RM::Paragraph.new('l1')),
        @RM::ListItem.new(nil,
          @RM::Paragraph.new('l2'))])]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_bullet_utf_8
    str = <<-STR
* 新しい機能
    STR

    expected = [
      @RM::List.new(:BULLET, *[
        @RM::ListItem.new(nil,
          @RM::Paragraph.new('新しい機能'))])]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_bullet_verbatim_heading
    str = <<-STR
* l1
    v

= H
    STR

    expected = [
      @RM::List.new(:BULLET, *[
        @RM::ListItem.new(nil,
          @RM::Paragraph.new('l1'),
          @RM::Verbatim.new("v\n"))]),
      @RM::Heading.new(1, 'H')]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_bullet_heading
    str = <<-STR
* = l1
    STR

    expected = [
      @RM::List.new(:BULLET, *[
        @RM::ListItem.new(nil,
          @RM::Heading.new(1, 'l1'))])]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_bullet_indent
    str = <<-STR
* l1
  * l1.1
* l2
    STR

    expected = [
      @RM::List.new(:BULLET, *[
        @RM::ListItem.new(nil,
          @RM::Paragraph.new('l1'),
          @RM::List.new(:BULLET, *[
            @RM::ListItem.new(nil,
              @RM::Paragraph.new('l1.1'))])),
        @RM::ListItem.new(nil,
          @RM::Paragraph.new('l2'))])]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_bullet_paragraph
    str = <<-STR
now is
* l1
* l2
the time
    STR

    expected = [
      @RM::Paragraph.new('now is'),
      @RM::List.new(:BULLET, *[
        @RM::ListItem.new(nil,
          @RM::Paragraph.new('l1')),
        @RM::ListItem.new(nil,
          @RM::Paragraph.new('l2')),
      ]),
      @RM::Paragraph.new('the time'),
    ]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_bullet_multiline
    str = <<-STR
* l1
  l1+
* l2
    STR

    expected = [
      list(:BULLET,
        item(nil,
          para('l1 ', 'l1+')),
        item(nil,
          para('l2')))]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_bullet_multiparagraph
    str = <<-STR
* l1

  l1+
    STR

    expected = [
      @RM::List.new(:BULLET, *[
        @RM::ListItem.new(nil,
          @RM::Paragraph.new('l1'),
          @RM::BlankLine.new,
          @RM::Paragraph.new('l1+')),
      ]),
    ]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_bullet_indent_verbatim
    str = <<-STR
* l1
  * l1.1
    text
      code
        code

    text
* l2
    STR

    expected = [
      list(:BULLET,
        item(nil,
          para('l1'),
          list(:BULLET,
            item(nil,
              para('l1.1 ', 'text'),
              verb("code\n", "  code\n"),
              para('text')))),
        item(nil,
          para('l2')))]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_dash
    str = <<-STR
- one
- two
    STR

    expected = [
      @RM::List.new(:BULLET, *[
        @RM::ListItem.new(nil,
          @RM::Paragraph.new('one')),
        @RM::ListItem.new(nil,
          @RM::Paragraph.new('two'))])]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_heading
    str = '= heading one'

    expected = [
      @RM::Heading.new(1, 'heading one')]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_heading_three
    str = '=== heading three'

    expected = [
      @RM::Heading.new(3, 'heading three')]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_heading_bullet
    str = '= * heading one'

    expected = [
      @RM::Heading.new(1, '* heading one')]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_heading_empty
    str = <<-STR
===
* bullet
    STR

    expected = [
      @RM::Heading.new(3, ''),
      @RM::BlankLine.new,
      @RM::List.new(:BULLET, *[
        @RM::ListItem.new(nil,
          @RM::Paragraph.new('bullet'))]),
    ]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_heading_heading
    str = '= ='

    expected = [
      @RM::Heading.new(1, '=')]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_heading_lalpha
    str = '= b. heading one'

    expected = [
      @RM::Heading.new(1, 'b. heading one')]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_heading_label
    str = '= [heading one]'

    expected = [
      @RM::Heading.new(1, '[heading one]')]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_heading_note
    str = '= heading one::'

    expected = [
      @RM::Heading.new(1, 'heading one::')]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_heading_number
    str = '= 5. heading one'

    expected = [
      @RM::Heading.new(1, '5. heading one')]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_heading_ualpha
    str = '= B. heading one'

    expected = [
      @RM::Heading.new(1, 'B. heading one')]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_label
    str = <<-STR
[one] item one
[two] item two
    STR

    expected = [
      list(:LABEL,
        item(%w[one],
          para('item one')),
        item(%w[two],
          para('item two')))]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_label_bullet
    str = <<-STR
[cat] l1
      * l1.1
[dog] l2
    STR

    expected = [
      list(:LABEL,
        item(%w[cat],
          para('l1'),
          list(:BULLET,
            item(nil,
              para('l1.1')))),
        item(%w[dog],
          para('l2')))]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_label_multi_label
    str = <<-STR
[one]
[two] some description
    STR

    expected = [
      list(:LABEL,
        item(%w[one two],
          para('some description')))]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_label_multi_line
    str = <<-STR
[cat] l1
      continuation
[dog] l2
    STR

    expected = [
      list(:LABEL,
        item(%w[cat],
          para('l1 ', 'continuation')),
        item(%w[dog],
          para('l2')))]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_label_newline
    str = <<-STR
[one]
  item one
[two]
  item two
    STR

    expected = [
      list(:LABEL,
        item(%w[one],
          para('item one')),
        item(%w[two],
          para('item two')))]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_lalpha
    str = <<-STR
a. l1
b. l2
    STR

    expected = [
      @RM::List.new(:LALPHA, *[
        @RM::ListItem.new(nil,
          @RM::Paragraph.new('l1')),
        @RM::ListItem.new(nil,
          @RM::Paragraph.new('l2'))])]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_lalpha_ualpha
    str = <<-STR
a. l1
b. l2
A. l3
A. l4
    STR

    expected = [
      @RM::List.new(:LALPHA, *[
        @RM::ListItem.new(nil,
          @RM::Paragraph.new('l1')),
        @RM::ListItem.new(nil,
          @RM::Paragraph.new('l2'))]),
      @RM::List.new(:UALPHA, *[
        @RM::ListItem.new(nil,
          @RM::Paragraph.new('l3')),
        @RM::ListItem.new(nil,
          @RM::Paragraph.new('l4'))])]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_lalpha_utf_8
    str = <<-STR
a. 新しい機能
    STR

    expected = [
      @RM::List.new(:LALPHA, *[
        @RM::ListItem.new(nil,
          @RM::Paragraph.new('新しい機能'))])]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_line_break
    str = "now is\nthe time  \nfor all"

    expected = [
      para('now is ', 'the time'),
      blank_line,
      para('for all')]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_list_list_1
    str = <<-STR
10. para 1

    [label 1]
      para 1.1

        code

      para 1.2
    STR

    expected = [
      list(:NUMBER,
        item(nil,
          para('para 1'),
          blank_line,
          list(:LABEL,
            item(%w[label\ 1],
              para('para 1.1'),
              blank_line,
              verb("code\n"),
              para('para 1.2')))))]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_list_list_2
    str = <<-STR
6. para

   label 1::  text 1
   label 2::  text 2
    STR

    expected = [
      list(:NUMBER,
        item(nil,
          para('para'),
          blank_line,
          list(:NOTE,
            item(%w[label\ 1],
              para('text 1')),
            item(%w[label\ 2],
              para('text 2')))))]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_list_verbatim
    str = <<-STR
* one
    verb1
    verb2
* two
    STR

    expected = [
      @RM::List.new(:BULLET, *[
        @RM::ListItem.new(nil,
          @RM::Paragraph.new('one'),
          @RM::Verbatim.new("verb1\n", "verb2\n")),
        @RM::ListItem.new(nil,
          @RM::Paragraph.new('two'))])]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_lists
    str = <<-STR
now is
* l1
1. n1
2. n2
* l2
the time
    STR

    expected = [
      @RM::Paragraph.new('now is'),
      @RM::List.new(:BULLET, *[
        @RM::ListItem.new(nil,
          @RM::Paragraph.new('l1'))]),
      @RM::List.new(:NUMBER, *[
        @RM::ListItem.new(nil,
          @RM::Paragraph.new('n1')),
        @RM::ListItem.new(nil,
          @RM::Paragraph.new('n2'))]),
      @RM::List.new(:BULLET, *[
        @RM::ListItem.new(nil,
          @RM::Paragraph.new('l2'))]),
      @RM::Paragraph.new('the time')]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_note
    str = <<-STR
one:: item one
two:: item two
    STR

    expected = [
      list(:NOTE,
        item(%w[one],
          para('item one')),
        item(%w[two],
          para('item two')))]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_note_empty
    str = <<-STR
one::
two::
    STR

    expected = [
      list(:NOTE,
        item(%w[one two],
          blank_line))]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_note_note
    str = <<-STR
one:: two::
    STR

    expected = [
      list(:NOTE,
        item(%w[one],
          list(:NOTE,
            item(%w[two],
              blank_line))))]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_number_bullet
    str = <<-STR
1. l1
   * l1.1
2. l2
    STR

    expected = [
      list(:NUMBER,
        item(nil,
          para('l1'),
          list(:BULLET,
            item(nil,
              para('l1.1')))),
        item(nil,
          para('l2')))]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_paragraph
    str = <<-STR
now is the time

for all good men
    STR

    expected = [
      @RM::Paragraph.new('now is the time'),
      @RM::BlankLine.new,
      @RM::Paragraph.new('for all good men')]
    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_paragraph_multiline
    str = "now is the time\nfor all good men"

    expected = @RM::Paragraph.new 'now is the time ', 'for all good men'
    assert_equal [expected], @RMP.parse(str).parts
  end

  def test_parse_paragraph_verbatim
    str = <<-STR
now is the time
  code _line_ here
for all good men
    STR

    expected = [
      @RM::Paragraph.new('now is the time'),
      @RM::Verbatim.new("code _line_ here\n"),
      @RM::Paragraph.new('for all good men'),
    ]
    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_rule
    str = <<-STR
now is the time

---

for all good men
    STR

    expected = [
      @RM::Paragraph.new('now is the time'),
      @RM::BlankLine.new,
      @RM::Rule.new(1),
      @RM::BlankLine.new,
      @RM::Paragraph.new('for all good men')]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_ualpha
    str = <<-STR
A. l1
B. l2
    STR

    expected = [
      @RM::List.new(:UALPHA, *[
        @RM::ListItem.new(nil,
          @RM::Paragraph.new('l1')),
        @RM::ListItem.new(nil,
          @RM::Paragraph.new('l2'))])]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_trailing_cr
    expected = [ @RM::Paragraph.new('Text') ]
    # FIXME hangs the parser:
    assert_equal expected, @RMP.parse("Text\r").parts
  end

  def test_parse_verbatim
    str = <<-STR
now is
   code
the time
    STR

    expected = [
      @RM::Paragraph.new('now is'),
      @RM::Verbatim.new("code\n"),
      @RM::Paragraph.new('the time'),
    ]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_verbatim_bullet
    str = <<-STR
  * blah
    STR

    expected = [
      @RM::Verbatim.new("* blah\n")]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_verbatim_dash
    str = <<-STR
  - blah
    STR

    expected = [
      @RM::Verbatim.new("- blah\n")]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_verbatim_fold
    str = <<-STR
now is
   code


   code1

the time
    STR

    expected = [
      @RM::Paragraph.new('now is'),
      @RM::Verbatim.new("code\n", "\n", "code1\n"),
      @RM::Paragraph.new('the time'),
    ]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_verbatim_heading
    str = <<-STR
text
   ===   heading three
    STR

    expected = [
      @RM::Paragraph.new('text'),
      @RM::Verbatim.new("===   heading three\n")]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_verbatim_heading2
    str = "text\n   code\n=== heading three"

    expected = [
      @RM::Paragraph.new('text'),
      @RM::Verbatim.new("code\n"),
      @RM::Heading.new(3, 'heading three')]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_verbatim_label
    str = <<-STR
  [blah] blah
    STR

    expected = [
      @RM::Verbatim.new("[blah] blah\n")]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_verbatim_lalpha
    str = <<-STR
  b. blah
    STR

    expected = [
      @RM::Verbatim.new("b. blah\n")]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_verbatim_markup_example
    str = <<-STR
text
   code
   === heading three
    STR

    expected = [
      @RM::Paragraph.new('text'),
      @RM::Verbatim.new("code\n", "=== heading three\n")]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_verbatim_merge
    str = <<-STR
now is
   code

   code1
the time
    STR

    expected = [
      @RM::Paragraph.new('now is'),
      @RM::Verbatim.new("code\n", "\n", "code1\n"),
      @RM::Paragraph.new('the time'),
    ]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_verbatim_merge2
    str = <<-STR
now is
   code

   code1

   code2
the time
    STR

    expected = [
      @RM::Paragraph.new('now is'),
      @RM::Verbatim.new("code\n", "\n", "code1\n", "\n", "code2\n"),
      @RM::Paragraph.new('the time'),
    ]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_verbatim_multiline
    str = <<-STR
now is
   code
   code1
the time
    STR

    expected = [
      @RM::Paragraph.new('now is'),
      @RM::Verbatim.new("code\n", "code1\n"),
      @RM::Paragraph.new('the time'),
    ]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_verbatim_multilevel
    str = <<-STR
now is the time
  code
 more code
for all good men
    STR

    expected = [
      @RM::Paragraph.new('now is the time'),
      @RM::Verbatim.new(" code\n",
                        "more code\n"),
      @RM::Paragraph.new('for all good men'),
    ]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_verbatim_note
    str = <<-STR
  blah:: blah
    STR

    expected = [
      @RM::Verbatim.new("blah:: blah\n")]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_verbatim_number
    str = <<-STR
  2. blah
    STR

    expected = [
      @RM::Verbatim.new("2. blah\n")]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_verbatim_rule
    str = <<-STR
text

  --- lib/blah.rb.orig
  +++ lib/blah.rb
    STR

    expected = [
      @RM::Paragraph.new('text'),
      @RM::BlankLine.new,
      @RM::Verbatim.new("--- lib/blah.rb.orig\n",
                        "+++ lib/blah.rb\n")]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_verbatim_rule2
    str = <<-STR.chomp
text

  ---
    STR

    expected = [
      @RM::Paragraph.new('text'),
      @RM::BlankLine.new,
      @RM::Verbatim.new("---")]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_verbatim_trim
    str = <<-STR
now is
   code

   code1

the time
    STR

    expected = [
      @RM::Paragraph.new('now is'),
      @RM::Verbatim.new("code\n",
                        "\n",
                        "code1\n"),
      @RM::Paragraph.new('the time'),
    ]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_verbatim_ualpha
    str = <<-STR
  B. blah
    STR

    expected = [
      @RM::Verbatim.new("B. blah\n")]

    assert_equal expected, @RMP.parse(str).parts
  end

  def test_parse_whitespace
    expected = [
      @RM::Paragraph.new('hello'),
    ]

    assert_equal expected, @RMP.parse('hello').parts

    expected = [
      @RM::Verbatim.new('hello '),
    ]

    assert_equal expected, @RMP.parse('  hello ').parts

    expected = [
      @RM::Verbatim.new('hello          '),
    ]

    assert_equal expected, @RMP.parse('                 hello          ').parts

    expected = [
      @RM::Paragraph.new('1'),
      @RM::Verbatim.new("2\n", ' 3'),
    ]

    assert_equal expected, @RMP.parse("1\n 2\n  3").parts

    expected = [
      @RM::Verbatim.new("1\n",
                        " 2\n",
                        "  3"),
    ]

    assert_equal expected, @RMP.parse("  1\n   2\n    3").parts

    expected = [
      @RM::Paragraph.new('1'),
      @RM::Verbatim.new("2\n",
                        " 3\n"),
      @RM::Paragraph.new('1'),
      @RM::Verbatim.new('2'),
    ]

    assert_equal expected, @RMP.parse("1\n 2\n  3\n1\n 2").parts

    expected = [
      @RM::Verbatim.new("1\n",
                        " 2\n",
                        "  3\n",
                        "1\n",
                        ' 2'),
    ]

    assert_equal expected, @RMP.parse("  1\n   2\n    3\n  1\n   2").parts

    expected = [
      @RM::Verbatim.new("1\n",
                        " 2\n",
                        "\n",
                        '  3'),
    ]

    assert_equal expected, @RMP.parse("  1\n   2\n\n    3").parts
  end

  def test_peek_token
    parser = util_parser

    assert_equal [:HEADER, 1, 0, 0], parser.peek_token

    assert_equal 8, parser.tokens.length
  end

  def test_skip
    parser = util_parser

    assert_equal [:HEADER, 1, 0, 0], parser.skip(:HEADER)

    assert_equal [:TEXT, 'Heading', 2, 0], parser.get

    assert_equal [:NEWLINE, "\n", 9, 0], parser.peek_token

    assert_raises RDoc::Markup::Parser::ParseError do
      parser.skip :NONE
    end

    assert_equal [:NEWLINE, "\n", 9, 0], parser.peek_token

    assert_equal nil, parser.skip(:NONE, false)

    assert_equal [:NEWLINE, "\n", 9, 0], parser.peek_token
  end

  def test_tokenize_bullet
    str = <<-STR
* l1
    STR

    expected = [
      [:BULLET,  '*',     0, 0],
      [:TEXT,    'l1',    2, 0],
      [:NEWLINE, "\n",    4, 0],
    ]

    assert_equal expected, @RMP.tokenize(str)
  end

  def test_tokenize_bullet_indent
    str = <<-STR
* l1
  * l1.1
    STR

    expected = [
      [:BULLET,  '*',     0, 0],
      [:TEXT,    'l1',    2, 0],
      [:NEWLINE, "\n",    4, 0],
      [:BULLET,  '*',     2, 1],
      [:TEXT,    'l1.1',  4, 1],
      [:NEWLINE, "\n",    8, 1],
    ]

    assert_equal expected, @RMP.tokenize(str)
  end

  def test_tokenize_heading
    str = <<-STR
= Heading
== Heading 2
    STR

    expected = [
      [:HEADER,  1,            0, 0],
      [:TEXT,    'Heading',    2, 0],
      [:NEWLINE, "\n",         9, 0],
      [:HEADER,  2,            0, 1],
      [:TEXT,    'Heading 2',  3, 1],
      [:NEWLINE, "\n",        12, 1],
    ]

    assert_equal expected, @RMP.tokenize(str)
  end

  def test_tokenize_heading_empty
    str = <<-STR
===
* bullet
    STR

    expected = [
      [:HEADER,  3,        0, 0],
      [:NEWLINE, "\n",     3, 0],
      [:BULLET,  "*",      0, 1],
      [:TEXT,    "bullet", 2, 1],
      [:NEWLINE, "\n",     8, 1],
    ]

    assert_equal expected, @RMP.tokenize(str)
  end

  def test_tokenize_heading_heading
    str = <<-STR
= =
    STR

    expected = [
      [:HEADER,  1,    0, 0],
      [:TEXT,    '=',  2, 0],
      [:NEWLINE, "\n", 3, 0],
    ]

    assert_equal expected, @RMP.tokenize(str)
  end

  def test_tokenize_heading_no_space
    str = <<-STR
=Heading
==Heading 2
    STR

    expected = [
      [:HEADER,  1,            0, 0],
      [:TEXT,    'Heading',    1, 0],
      [:NEWLINE, "\n",         8, 0],
      [:HEADER,  2,            0, 1],
      [:TEXT,    'Heading 2',  2, 1],
      [:NEWLINE, "\n",        11, 1],
    ]

    assert_equal expected, @RMP.tokenize(str)
  end

  def test_tokenize_label
    str = <<-STR
[cat] l1
[dog] l1.1
    STR

    expected = [
      [:LABEL,   'cat',   0, 0],
      [:TEXT,    'l1',    6, 0],
      [:NEWLINE, "\n",    8, 0],
      [:LABEL,   'dog',   0, 1],
      [:TEXT,    'l1.1',  6, 1],
      [:NEWLINE, "\n",   10, 1],
    ]

    assert_equal expected, @RMP.tokenize(str)
  end

  def test_tokenize_label_note
    str = <<-STR
[label]
  note::
    STR

    expected = [
      [:LABEL,   'label', 0, 0],
      [:NEWLINE, "\n",    7, 0],
      [:NOTE,    'note',  2, 1],
      [:NEWLINE, "\n",    8, 1],
    ]

    assert_equal expected, @RMP.tokenize(str)
  end

  def test_tokenize_label_newline
    str = <<-STR
[cat]
  l1
    STR

    expected = [
      [:LABEL,   'cat',   0, 0],
      [:NEWLINE, "\n",    5, 0],
      [:TEXT,    'l1',    2, 1],
      [:NEWLINE, "\n",    4, 1],
    ]

    assert_equal expected, @RMP.tokenize(str)
  end

  def test_tokenize_label_newline_windows
    str = <<-STR
[cat]\r
  l1\r
    STR

    expected = [
      [:LABEL,   'cat',   0, 0],
      [:NEWLINE, "\n",    6, 0],
      [:TEXT,    'l1',    2, 1],
      [:NEWLINE, "\n",    5, 1],
    ]

    assert_equal expected, @RMP.tokenize(str)
  end

  def test_tokenize_lalpha
    str = <<-STR
a. l1
b. l1.1
    STR

    expected = [
      [:LALPHA,  'a',     0, 0],
      [:TEXT,    'l1',    3, 0],
      [:NEWLINE, "\n",    5, 0],
      [:LALPHA,  'b',     0, 1],
      [:TEXT,    'l1.1',  3, 1],
      [:NEWLINE, "\n",    7, 1],
    ]

    assert_equal expected, @RMP.tokenize(str)
  end

  def test_tokenize_line_break
    str = "now is\nthe time  \nfor all\n"

    expected = [
      [:TEXT,    'now is',    0, 0],
      [:NEWLINE, "\n",        6, 0],
      [:TEXT,    'the time',  0, 1],
      [:BREAK,   "  ",        8, 1],
      [:NEWLINE, "\n",       10, 1],
      [:TEXT,    'for all',   0, 2],
      [:NEWLINE, "\n",        7, 2],
    ]

    assert_equal expected, @RMP.tokenize(str)
  end

  def test_tokenize_line_break_long
    str = "now is\nthe time   \nfor all\n"

    expected = [
      [:TEXT,    'now is',     0, 0],
      [:NEWLINE, "\n",         6, 0],
      [:TEXT,    'the time ',  0, 1],
      [:BREAK,   '  ',         9, 1],
      [:NEWLINE, "\n",        11, 1],
      [:TEXT,    'for all',    0, 2],
      [:NEWLINE, "\n",         7, 2],
    ]

    assert_equal expected, @RMP.tokenize(str)
  end

  def test_tokenize_line_break_no_short
    str = "now is\nthe time \nfor all\n"

    expected = [
      [:TEXT,    'now is',    0, 0],
      [:NEWLINE, "\n",        6, 0],
      [:TEXT,    'the time ', 0, 1],
      [:NEWLINE, "\n",        9, 1],
      [:TEXT,    'for all',   0, 2],
      [:NEWLINE, "\n",        7, 2],
    ]

    assert_equal expected, @RMP.tokenize(str)
  end

  def test_tokenize_note
    str = <<-STR
cat:: l1
dog:: l1.1
    STR

    expected = [
      [:NOTE,    'cat',   0, 0],
      [:TEXT,    'l1',    6, 0],
      [:NEWLINE, "\n",    8, 0],
      [:NOTE,    'dog',   0, 1],
      [:TEXT,    'l1.1',  6, 1],
      [:NEWLINE, "\n",   10, 1],
    ]

    assert_equal expected, @RMP.tokenize(str)
  end

  def test_tokenize_note_empty
    str = <<-STR
cat::
dog::
    STR

    expected = [
      [:NOTE,    'cat', 0, 0],
      [:NEWLINE, "\n",  5, 0],
      [:NOTE,    'dog', 0, 1],
      [:NEWLINE, "\n",  5, 1],
    ]

    assert_equal expected, @RMP.tokenize(str)
  end

  def test_tokenize_note_newline
    str = <<-STR
cat::
  l1
    STR

    expected = [
      [:NOTE,    'cat',   0, 0],
      [:NEWLINE, "\n",    5, 0],
      [:TEXT,    'l1',    2, 1],
      [:NEWLINE, "\n",    4, 1],
    ]

    assert_equal expected, @RMP.tokenize(str)
  end

  def test_tokenize_note_utf_8
    skip 'Encoding not implemented' unless @have_encoding

    str = <<-STR
cät:: l1a
      l1b
døg:: l2a
      l2b
    STR

    expected = [
      [:NOTE,    'cät',   0, 0],
      [:TEXT,    'l1a',   6, 0],
      [:NEWLINE, "\n",    9, 0],
      [:TEXT,    'l1b',   6, 1],
      [:NEWLINE, "\n",    9, 1],
      [:NOTE,    'døg',   0, 2],
      [:TEXT,    'l2a',   6, 2],
      [:NEWLINE, "\n",    9, 2],
      [:TEXT,    'l2b',   6, 3],
      [:NEWLINE, "\n",    9, 3],
    ]

    assert_equal expected, @RMP.tokenize(str)
  end

  def test_tokenize_note_newline_windows
    str = <<-STR
cat::\r
  l1\r
    STR

    expected = [
      [:NOTE,    'cat',   0, 0],
      [:NEWLINE, "\n",    6, 0],
      [:TEXT,    'l1',    2, 1],
      [:NEWLINE, "\n",    5, 1],
    ]

    assert_equal expected, @RMP.tokenize(str)
  end

  def test_tokenize_note_not
    str = <<-STR
Cat::Dog
    STR

    expected = [
      [:TEXT,    'Cat::Dog', 0, 0],
      [:NEWLINE, "\n",       8, 0],
    ]

    assert_equal expected, @RMP.tokenize(str)
  end

  def test_tokenize_number
    str = <<-STR
1. l1
2. l1.1
    STR

    expected = [
      [:NUMBER,  '1',     0, 0],
      [:TEXT,    'l1',    3, 0],
      [:NEWLINE, "\n",    5, 0],
      [:NUMBER,  '2',     0, 1],
      [:TEXT,    'l1.1',  3, 1],
      [:NEWLINE, "\n",    7, 1],
    ]

    assert_equal expected, @RMP.tokenize(str)
  end

  def test_tokenize_number_period
    str = <<-STR
1. blah blah blah
   l.
2. blah blah blah blah
   d.
    STR

    expected = [
      [:NUMBER,  "1",                    0, 0],
      [:TEXT,    "blah blah blah",       3, 0],
      [:NEWLINE, "\n",                  17, 0],

      [:TEXT,    "l.",                   3, 1],
      [:NEWLINE, "\n",                   5, 1],

      [:NUMBER,  "2",                    0, 2],
      [:TEXT,    "blah blah blah blah",  3, 2],
      [:NEWLINE, "\n",                  22, 2],

      [:TEXT,    "d.",                   3, 3],
      [:NEWLINE, "\n",                   5, 3]
    ]

    assert_equal expected, @RMP.tokenize(str)
  end

  def test_tokenize_number_period_continue
    str = <<-STR
1. blah blah blah
   l.  more stuff
2. blah blah blah blah
   d. other stuff
    STR

    expected = [
      [:NUMBER,  "1",                    0, 0],
      [:TEXT,    "blah blah blah",       3, 0],
      [:NEWLINE, "\n",                  17, 0],

      [:LALPHA,  "l",                    3, 1],
      [:TEXT,    "more stuff",           7, 1],
      [:NEWLINE, "\n",                  17, 1],

      [:NUMBER,  "2",                    0, 2],
      [:TEXT,    "blah blah blah blah",  3, 2],
      [:NEWLINE, "\n",                  22, 2],

      [:LALPHA,  "d",                    3, 3],
      [:TEXT,    "other stuff",          6, 3],
      [:NEWLINE, "\n",                  17, 3]
    ]

    assert_equal expected, @RMP.tokenize(str)
  end

  def test_tokenize_paragraphs
    str = <<-STR
now is
the time

for all
    STR

    expected = [
      [:TEXT,    'now is',   0, 0],
      [:NEWLINE, "\n",       6, 0],
      [:TEXT,    'the time', 0, 1],
      [:NEWLINE, "\n",       8, 1],
      [:NEWLINE, "\n",       0, 2],
      [:TEXT,    'for all',  0, 3],
      [:NEWLINE, "\n",       7, 3],
    ]

    assert_equal expected, @RMP.tokenize(str)
  end

  def test_tokenize_rule
    str = <<-STR
---

--- blah ---
    STR

    expected = [
      [:RULE,    1,            0, 0],
      [:NEWLINE, "\n",         3, 0],
      [:NEWLINE, "\n",         0, 1],
      [:TEXT, "--- blah ---",  0, 2],
      [:NEWLINE, "\n",        12, 2],
    ]

    assert_equal expected, @RMP.tokenize(str)
  end

  def test_tokenize_rule_windows
    str = <<-STR
---\r

--- blah ---\r
    STR

    expected = [
      [:RULE,    1,            0, 0],
      [:NEWLINE, "\n",         4, 0],
      [:NEWLINE, "\n",         0, 1],
      [:TEXT, "--- blah ---",  0, 2],
      [:NEWLINE, "\n",        13, 2],
    ]

    assert_equal expected, @RMP.tokenize(str)
  end

  def test_tokenize_ualpha
    str = <<-STR
A. l1
B. l1.1
    STR

    expected = [
      [:UALPHA,  'A',     0, 0],
      [:TEXT,    'l1',    3, 0],
      [:NEWLINE, "\n",    5, 0],
      [:UALPHA,  'B',     0, 1],
      [:TEXT,    'l1.1',  3, 1],
      [:NEWLINE, "\n",    7, 1],
    ]

    assert_equal expected, @RMP.tokenize(str)
  end

  def test_tokenize_verbatim_heading
    str = <<-STR
Example heading:

   === heading three
    STR

    expected = [
      [:TEXT,    'Example heading:',  0, 0],
      [:NEWLINE, "\n",               16, 0],
      [:NEWLINE, "\n",                0, 1],
      [:HEADER,  3,                   3, 2],
      [:TEXT,    'heading three',     7, 2],
      [:NEWLINE, "\n",               20, 2],
    ]

    assert_equal expected, @RMP.tokenize(str)
  end

  def test_tokenize_verbatim_rule
    str = <<-STR
  Verbatim section here that is double-underlined
  ===============================================
    STR

    expected = [
      [:TEXT,    'Verbatim section here that is double-underlined',  2, 0],
      [:NEWLINE, "\n",                                              49, 0],
      [:HEADER,  47,                                                 2, 1],
      [:NEWLINE, "\n",                                              49, 1],
    ]

    assert_equal expected, @RMP.tokenize(str)
  end

  def test_tokenize_verbatim_rule_fancy
    str = <<-STR
  A
    b
  ===============================================
    c
    STR

    expected = [
      [:TEXT,    'A',   2, 0],
      [:NEWLINE, "\n",  3, 0],
      [:TEXT,    'b',   4, 1],
      [:NEWLINE, "\n",  5, 1],
      [:HEADER,  47,    2, 2],
      [:NEWLINE, "\n", 49, 2],
      [:TEXT,    'c',   4, 3],
      [:NEWLINE, "\n",  5, 3],
    ]

    assert_equal expected, @RMP.tokenize(str)
  end

  def test_token_pos
    parser = @RMP.new
    s = parser.setup_scanner 'cät'

    s.scan(/\S+/)

    if @have_encoding or @have_byteslice then
      assert_equal [3, 0], parser.token_pos(s.pos)
    else
      assert_equal [4, 0], parser.token_pos(s.pos)
    end
  end

  # HACK move to Verbatim test case
  def test_verbatim_normalize
    v = @RM::Verbatim.new "foo\n", "\n", "\n", "bar\n"

    v.normalize

    assert_equal ["foo\n", "\n", "bar\n"], v.parts

    v = @RM::Verbatim.new "foo\n", "\n"

    v.normalize

    assert_equal ["foo\n"], v.parts
  end

  def test_unget
    parser = util_parser

    parser.get

    parser.unget

    assert_equal [:HEADER, 1, 0, 0], parser.peek_token

    assert_raises @RMP::Error do
      parser.unget
    end

    assert_equal 8, parser.tokens.length
  end

  def util_parser
    str = <<-STR
= Heading

Some text here
some more text over here
    STR

    @parser = @RMP.new
    @parser.tokenize str
    @parser
  end

end