diff --git a/lib/sidekiq/logging.rb b/lib/sidekiq/logging.rb
index 7dc6fe19..5156275a 100644
--- a/lib/sidekiq/logging.rb
+++ b/lib/sidekiq/logging.rb
@@ -17,12 +17,10 @@ module Sidekiq
     end
 
     def self.with_context(msg)
-      begin
-        Thread.current[:sidekiq_context] = msg
-        yield
-      ensure
-        Thread.current[:sidekiq_context] = nil
-      end
+      previous, Thread.current[:sidekiq_context] = Thread.current[:sidekiq_context], msg
+      yield
+    ensure
+      Thread.current[:sidekiq_context] = previous
     end
 
     def self.initialize_logger(log_target = STDOUT)
diff --git a/test/test_logging.rb b/test/test_logging.rb
new file mode 100644
index 00000000..da789039
--- /dev/null
+++ b/test/test_logging.rb
@@ -0,0 +1,34 @@
+require 'helper'
+require 'sidekiq/logging'
+
+class TestFetcher < Sidekiq::Test
+  describe Sidekiq::Logging do
+    describe "#with_context" do
+      def context
+        Sidekiq::Logging.logger.formatter.context
+      end
+
+      it "has no context by default" do
+        context.must_equal ""
+      end
+
+      it "can add a context" do
+        Sidekiq::Logging.with_context "xx" do
+          context.must_equal " xx"
+        end
+        context.must_equal ""
+      end
+
+      it "can use multiple contexts" do
+        Sidekiq::Logging.with_context "xx" do
+          context.must_equal " xx"
+          Sidekiq::Logging.with_context "yy" do
+            context.must_equal " yy"
+          end
+          context.must_equal " xx"
+        end
+        context.must_equal ""
+      end
+    end
+  end
+end