diff --git a/activesupport/lib/active_support/duration.rb b/activesupport/lib/active_support/duration.rb
index 4cce0a6967..fe87cbd436 100644
--- a/activesupport/lib/active_support/duration.rb
+++ b/activesupport/lib/active_support/duration.rb
@@ -403,8 +403,14 @@ module ActiveSupport
 
     private
       def sum(sign, time = ::Time.current)
-        parts.inject(time) do |t, (type, number)|
-          if t.acts_like?(:time) || t.acts_like?(:date)
+        unless time.acts_like?(:time) || time.acts_like?(:date)
+          raise ::ArgumentError, "expected a time or date, got #{time.inspect}"
+        end
+
+        if parts.empty?
+          time.since(sign * value)
+        else
+          parts.inject(time) do |t, (type, number)|
             if type == :seconds
               t.since(sign * number)
             elsif type == :minutes
@@ -414,8 +420,6 @@ module ActiveSupport
             else
               t.advance(type => sign * number)
             end
-          else
-            raise ::ArgumentError, "expected a time or date, got #{time.inspect}"
           end
         end
       end
diff --git a/activesupport/test/core_ext/duration_test.rb b/activesupport/test/core_ext/duration_test.rb
index f3608cf14d..ed4b0dd438 100644
--- a/activesupport/test/core_ext/duration_test.rb
+++ b/activesupport/test/core_ext/duration_test.rb
@@ -204,7 +204,9 @@ class DurationTest < ActiveSupport::TestCase
   def test_since_and_ago
     t = Time.local(2000)
     assert_equal t + 1, 1.second.since(t)
+    assert_equal t + 1, (1.minute / 60).since(t)
     assert_equal t - 1, 1.second.ago(t)
+    assert_equal t - 1, (1.minute / 60).ago(t)
   end
 
   def test_since_and_ago_without_argument