diff --git a/parse.y b/parse.y
index 8162086033..ab4f8ae233 100644
--- a/parse.y
+++ b/parse.y
@@ -5260,11 +5260,10 @@ parser_yyerror(struct parser_params *parser, const char *msg)
     int i;
 
     p = lex_p;
-    while (lex_pbeg <= p) {
-	if (*p == '\n') break;
+    while (lex_pbeg < p) {
+	if (*(p-1) == '\n') break;
 	p--;
     }
-    p++;
 
     pe = lex_p;
     while (pe < lex_pend) {
diff --git a/test/ruby/test_parse.rb b/test/ruby/test_parse.rb
index 57d03166c7..b4a63e369d 100644
--- a/test/ruby/test_parse.rb
+++ b/test/ruby/test_parse.rb
@@ -953,6 +953,11 @@ x = __ENCODING__
     end
   end
 
+  def test_yyerror_at_eol
+    assert_syntax_error("    0b", /\^/)
+    assert_syntax_error("    0b\n", /\^/)
+  end
+
 =begin
   def test_past_scope_variable
     assert_warning(/past scope/) {catch {|tag| eval("BEGIN{throw tag}; tap {a = 1}; a")}}