diff --git a/lib/reline/line_editor.rb b/lib/reline/line_editor.rb
index 71d765109d..8c0b858747 100644
--- a/lib/reline/line_editor.rb
+++ b/lib/reline/line_editor.rb
@@ -1175,7 +1175,12 @@ class Reline::LineEditor
           @history_pointer = nil
           hit = @line_backup_in_history
         else
-          hit_index = Reline::HISTORY.rindex { |item|
+          if @history_pointer
+            history = Reline::HISTORY[0..@history_pointer]
+          else
+            history = Reline::HISTORY
+          end
+          hit_index = history.rindex { |item|
             item.include?(search_word)
           }
           if hit_index
diff --git a/test/reline/test_key_actor_emacs.rb b/test/reline/test_key_actor_emacs.rb
index 5857f653cf..931e596299 100644
--- a/test/reline/test_key_actor_emacs.rb
+++ b/test/reline/test_key_actor_emacs.rb
@@ -1403,6 +1403,34 @@ class Reline::KeyActor::Emacs::Test < Reline::TestCase
     @config.history_size = history_size
   end
 
+  def test_search_history_to_back
+    setup_editor
+    Reline::HISTORY.concat([
+      '1235', # old
+      '12aa',
+      '1234' # new
+    ])
+    assert_line('')
+    assert_byte_pointer_size('')
+    assert_cursor(0)
+    assert_cursor_max(0)
+    input_keys("\C-r123")
+    assert_line('1234')
+    assert_byte_pointer_size('')
+    assert_cursor(0)
+    assert_cursor_max(0) # doesn't determine yet
+    input_keys("\C-ha")
+    assert_line('12aa')
+    assert_byte_pointer_size('')
+    assert_cursor(0)
+    assert_cursor_max(0)
+    input_keys("\C-h3")
+    assert_line('1235')
+    assert_byte_pointer_size('')
+    assert_cursor(0)
+    assert_cursor_max(0)
+  end
+
   def test_em_set_mark_and_em_exchange_mark
     input_keys('aaa bbb ccc ddd')
     assert_byte_pointer_size('aaa bbb ccc ddd')