diff --git a/lib/sidekiq/logger.rb b/lib/sidekiq/logger.rb
index c8af3203..035f2a34 100644
--- a/lib/sidekiq/logger.rb
+++ b/lib/sidekiq/logger.rb
@@ -6,10 +6,11 @@ require "time"
 module Sidekiq
   module Context
     def self.with(hash)
+      orig_context = current.dup
       current.merge!(hash)
       yield
     ensure
-      hash.each_key { |key| current.delete(key) }
+      Thread.current[:sidekiq_context] = orig_context
     end
 
     def self.current
diff --git a/test/test_logger.rb b/test/test_logger.rb
index 44ae0a63..fbbea0ab 100644
--- a/test/test_logger.rb
+++ b/test/test_logger.rb
@@ -49,6 +49,18 @@ class TestLogger < Minitest::Test
     assert_equal({}, subject.current)
   end
 
+  def test_with_overlapping_context
+    subject = Sidekiq::Context
+    subject.current.merge!({ foo: 'bar' })
+    assert_equal({ foo: 'bar' }, subject.current)
+
+    subject.with(foo: 'bingo') do
+      assert_equal({ foo: 'bingo' }, subject.current)
+    end
+
+    assert_equal({ foo: 'bar' }, subject.current)
+  end
+
   def test_nested_contexts
     subject = Sidekiq::Context
     assert_equal({}, subject.current)