From ec60205014287c3224964abd557a4c9891cd4a52 Mon Sep 17 00:00:00 2001 From: Jeremy Ashkenas Date: Wed, 30 Dec 2009 21:51:23 -0500 Subject: [PATCH] getting there, finally ... all tests are green for whitespace --- test/fixtures/generation/each.tokens | 2 +- test/unit/test_lexer.rb | 26 ++++++++++++++------------ 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/test/fixtures/generation/each.tokens b/test/fixtures/generation/each.tokens index 149a3170..aa5d853e 100644 --- a/test/fixtures/generation/each.tokens +++ b/test/fixtures/generation/each.tokens @@ -1 +1 @@ -[[:COMMENT, [" The cornerstone, an each implementation.", " Handles objects implementing forEach, arrays, and raw objects."]], ["\n", "\n"], [:IDENTIFIER, "_"], [:PROPERTY_ACCESS, "."], [:IDENTIFIER, "each"], [:ASSIGN, ":"], [:PARAM, "obj"], [",", ","], [:PARAM, "iterator"], [",", ","], [:PARAM, "context"], ["=>", "=>"], ["\n", "\n"], [:IDENTIFIER, "index"], [:ASSIGN, ":"], [:NUMBER, "0"], ["\n", "\n"], [:TRY, "try"], ["\n", "\n"], [:IF, "if"], [:IDENTIFIER, "obj"], [:PROPERTY_ACCESS, "."], [:IDENTIFIER, "forEach"], ["\n", "\n"], [:IDENTIFIER, "obj"], [:PROPERTY_ACCESS, "."], [:IDENTIFIER, "forEach"], ["(", "("], [:IDENTIFIER, "iterator"], [",", ","], [:IDENTIFIER, "context"], [")", ")"], ["\n", "\n"], [:ELSE, "else"], [:IF, "if"], [:IDENTIFIER, "_"], [:PROPERTY_ACCESS, "."], [:IDENTIFIER, "isArray"], ["(", "("], [:IDENTIFIER, "obj"], [")", ")"], [:OR, "or"], [:IDENTIFIER, "_"], [:PROPERTY_ACCESS, "."], [:IDENTIFIER, "isArguments"], ["(", "("], [:IDENTIFIER, "obj"], [")", ")"], ["\n", "\n"], [:IDENTIFIER, "iterator"], [:PROPERTY_ACCESS, "."], [:IDENTIFIER, "call"], ["(", "("], [:IDENTIFIER, "context"], [",", ","], [:IDENTIFIER, "item"], [",", ","], [:IDENTIFIER, "i"], [",", ","], [:IDENTIFIER, "obj"], [")", ")"], [:FOR, "for"], [:IDENTIFIER, "item"], [",", ","], [:IDENTIFIER, "i"], [:IN, "in"], [:IDENTIFIER, "obj"], [".", "."], ["\n", "\n"], [:ELSE, "else"], ["\n", "\n"], [:IDENTIFIER, "iterator"], [:PROPERTY_ACCESS, "."], [:IDENTIFIER, "call"], ["(", "("], [:IDENTIFIER, "context"], [",", ","], [:IDENTIFIER, "obj"], ["[", "["], [:IDENTIFIER, "key"], ["]", "]"], [",", ","], [:IDENTIFIER, "key"], [",", ","], [:IDENTIFIER, "obj"], [")", ")"], [:FOR, "for"], [:IDENTIFIER, "key"], [:IN, "in"], [:IDENTIFIER, "_"], [:PROPERTY_ACCESS, "."], [:IDENTIFIER, "keys"], ["(", "("], [:IDENTIFIER, "obj"], [")", ")"], [".", "."], [".", "."], ["\n", "\n"], [:CATCH, "catch"], [:IDENTIFIER, "e"], ["\n", "\n"], [:THROW, "throw"], [:IDENTIFIER, "e"], [:IF, "if"], [:IDENTIFIER, "e"], [:ISNT, "isnt"], [:IDENTIFIER, "breaker"], [".", "."], ["\n", "\n"], [:IDENTIFIER, "obj"], [".", "."]] \ No newline at end of file +[[:COMMENT, [" The cornerstone, an each implementation.", " Handles objects implementing forEach, arrays, and raw objects."]], ["\n", "\n"], [:IDENTIFIER, "_"], [:PROPERTY_ACCESS, "."], [:IDENTIFIER, "each"], [:ASSIGN, ":"], [:PARAM, "obj"], [",", ","], [:PARAM, "iterator"], [",", ","], [:PARAM, "context"], ["=>", "=>"], [:INDENT, 2], [:IDENTIFIER, "index"], [:ASSIGN, ":"], [:NUMBER, "0"], ["\n", "\n"], [:TRY, "try"], [:INDENT, 2], [:IF, "if"], [:IDENTIFIER, "obj"], [:PROPERTY_ACCESS, "."], [:IDENTIFIER, "forEach"], [:INDENT, 2], [:IDENTIFIER, "obj"], [:PROPERTY_ACCESS, "."], [:IDENTIFIER, "forEach"], ["(", "("], [:IDENTIFIER, "iterator"], [",", ","], [:IDENTIFIER, "context"], [")", ")"], [:OUTDENT, 2], [:ELSE, "else"], [:IF, "if"], [:IDENTIFIER, "_"], [:PROPERTY_ACCESS, "."], [:IDENTIFIER, "isArray"], ["(", "("], [:IDENTIFIER, "obj"], [")", ")"], [:OR, "or"], [:IDENTIFIER, "_"], [:PROPERTY_ACCESS, "."], [:IDENTIFIER, "isArguments"], ["(", "("], [:IDENTIFIER, "obj"], [")", ")"], [:INDENT, 2], [:IDENTIFIER, "iterator"], [:PROPERTY_ACCESS, "."], [:IDENTIFIER, "call"], ["(", "("], [:IDENTIFIER, "context"], [",", ","], [:IDENTIFIER, "item"], [",", ","], [:IDENTIFIER, "i"], [",", ","], [:IDENTIFIER, "obj"], [")", ")"], [:FOR, "for"], [:IDENTIFIER, "item"], [",", ","], [:IDENTIFIER, "i"], [:IN, "in"], [:IDENTIFIER, "obj"], [:OUTDENT, 2], [:ELSE, "else"], [:INDENT, 2], [:IDENTIFIER, "iterator"], [:PROPERTY_ACCESS, "."], [:IDENTIFIER, "call"], ["(", "("], [:IDENTIFIER, "context"], [",", ","], [:IDENTIFIER, "obj"], ["[", "["], [:IDENTIFIER, "key"], ["]", "]"], [",", ","], [:IDENTIFIER, "key"], [",", ","], [:IDENTIFIER, "obj"], [")", ")"], [:FOR, "for"], [:IDENTIFIER, "key"], [:IN, "in"], [:IDENTIFIER, "_"], [:PROPERTY_ACCESS, "."], [:IDENTIFIER, "keys"], ["(", "("], [:IDENTIFIER, "obj"], [")", ")"], [:OUTDENT, 2], [:OUTDENT, 2], [:CATCH, "catch"], [:IDENTIFIER, "e"], [:INDENT, 2], [:THROW, "throw"], [:IDENTIFIER, "e"], [:IF, "if"], [:IDENTIFIER, "e"], [:ISNT, "isnt"], [:IDENTIFIER, "breaker"], [:OUTDENT, 2], ["\n", "\n"], [:IDENTIFIER, "obj"], [:OUTDENT, 2], ["\n", "\n"]] \ No newline at end of file diff --git a/test/unit/test_lexer.rb b/test/unit/test_lexer.rb index 0d2ec1b5..5a811d34 100644 --- a/test/unit/test_lexer.rb +++ b/test/unit/test_lexer.rb @@ -7,45 +7,47 @@ class LexerTest < Test::Unit::TestCase end def test_lexing_an_empty_string - assert @lex.tokenize("") == [] + assert @lex.tokenize("") == [["\n", "\n"]] end def test_lexing_basic_assignment - code = "a: 'one'; b: [1, 2]" - assert @lex.tokenize(code) == [[:IDENTIFIER, "a"], [:ASSIGN, ":"], - [:STRING, "'one'"], [";", ";"], [:IDENTIFIER, "b"], [:ASSIGN, ":"], - ["[", "["], [:NUMBER, "1"], [",", ","], [:NUMBER, "2"], ["]", "]"]] + code = "a: 'one'\nb: [1, 2]" + assert @lex.tokenize(code) == [[:IDENTIFIER, "a"], [:ASSIGN, ":"], + [:STRING, "'one'"], ["\n", "\n"], [:IDENTIFIER, "b"], [:ASSIGN, ":"], + ["[", "["], [:NUMBER, "1"], [",", ","], [:NUMBER, "2"], ["]", "]"], + ["\n", "\n"]] end def test_lexing_object_literal code = "{one : 1}" assert @lex.tokenize(code) == [["{", "{"], [:IDENTIFIER, "one"], [:ASSIGN, ":"], - [:NUMBER, "1"], ["}", "}"]] + [:NUMBER, "1"], ["}", "}"], ["\n", "\n"]] end def test_lexing_function_definition - code = "x, y => x * y." + code = "x, y => x * y" assert @lex.tokenize(code) == [[:PARAM, "x"], [",", ","], [:PARAM, "y"], - ["=>", "=>"], [:IDENTIFIER, "x"], ["*", "*"], [:IDENTIFIER, "y"], [".", "."]] + ["=>", "=>"], [:INDENT, 2], [:IDENTIFIER, "x"], ["*", "*"], + [:IDENTIFIER, "y"], [:OUTDENT, 2], ["\n", "\n"]] end def test_lexing_if_statement code = "clap_your_hands() if happy" assert @lex.tokenize(code) == [[:IDENTIFIER, "clap_your_hands"], ["(", "("], - [")", ")"], [:IF, "if"], [:IDENTIFIER, "happy"]] + [")", ")"], [:IF, "if"], [:IDENTIFIER, "happy"], ["\n", "\n"]] end def test_lexing_comment - code = "a: 1\n # comment\n # on two lines\nb: 2" + code = "a: 1\n# comment\n# on two lines\nb: 2" assert @lex.tokenize(code) == [[:IDENTIFIER, "a"], [:ASSIGN, ":"], [:NUMBER, "1"], ["\n", "\n"], [:COMMENT, [" comment", " on two lines"]], ["\n", "\n"], - [:IDENTIFIER, "b"], [:ASSIGN, ":"], [:NUMBER, "2"]] + [:IDENTIFIER, "b"], [:ASSIGN, ":"], [:NUMBER, "2"], ["\n", "\n"]] end def test_lexing_newline_escaper code = "two: 1 + \\\n\n 1" assert @lex.tokenize(code) == [[:IDENTIFIER, "two"], [:ASSIGN, ":"], - [:NUMBER, "1"], ["+", "+"], [:NUMBER, "1"]] + [:NUMBER, "1"], ["+", "+"], [:NUMBER, "1"], ["\n", "\n"]] end def test_lexing