Improved "copy" strategy, including support for local caching and pattern exclusion

git-svn-id: http://svn.rubyonrails.org/rails/tools/capistrano@8993 5ecf4fe2-1ee6-0310-87b1-e25e094e27de

CHANGELOG

@@ -1,3 +1,8 @@
+*SVN*
+
+* Improved "copy" strategy supports local caching and pattern exclusion (via :copy_cache and :copy_exclude variables) [Jamis Buck]
+
+
 *2.2.0* February 27, 2008
 
 * Fix git submodule support to init on sync [halorgium]

lib/capistrano/recipes/deploy/strategy/base.rb

@@ -46,6 +46,12 @@ module Capistrano
             end
           end
 
+          # A wrapper for Kernel#system that logs the command being executed.
+          def system(*args)
+            logger.trace "executing locally: #{args.join(' ')}"
+            super
+          end
+
         private
 
           def logger

lib/capistrano/recipes/deploy/strategy/copy.rb

@@ -15,7 +15,26 @@ module Capistrano
       # of the source code. If you would rather use the export operation,
       # you can set the :copy_strategy variable to :export.
       #
-      # This deployment strategy supports a special variable,
+      #   set :copy_strategy, :export
+      #
+      # For even faster deployments, you can set the :copy_cache variable to
+      # true. This will cause deployments to do a new checkout of your
+      # repository to a new directory, and then copy that checkout. Subsequent
+      # deploys will just resync that copy, rather than doing an entirely new
+      # checkout. Additionally, you can specify file patterns to exclude from
+      # the copy when using :copy_cache; just set the :copy_exclude variable
+      # to an array of file globs or regexps.
+      #
+      #   set :copy_cache, true
+      #   set :copy_exclude, ".git/*"
+      #
+      # Note that :copy_strategy is ignored when :copy_cache is set. Also, if
+      # you want the copy cache put somewhere specific, you can set the variable
+      # to the path you want, instead of merely 'true':
+      #
+      #   set :copy_cache, "/tmp/caches/myapp"
+      #
+      # This deployment strategy also supports a special variable,
       # :copy_compression, which must be one of :gzip, :bz2, or
       # :zip, and which specifies how the source should be compressed for
       # transmission to each host.
@@ -25,8 +44,37 @@ module Capistrano
         # servers, and uncompresses it on each of them into the deployment
         # directory.
         def deploy!
+          if copy_cache
+            if File.exists?(copy_cache)
+              logger.debug "refreshing local cache to revision #{revision} at #{copy_cache}"
+              system(source.sync(revision, copy_cache))
+            else
+              logger.debug "preparing local cache at #{copy_cache}"
+              system(source.checkout(revision, copy_cache))
+            end
+
+            logger.debug "copying cache to deployment staging area #{destination}"
+            Dir.chdir(copy_cache) do
+              FileUtils.mkdir_p(destination)
+              queue = Dir.glob("*", File::FNM_DOTMATCH)
+              while queue.any?
+                item = queue.shift
+                name = File.basename(item)
+                next if name == "." || name == ".."
+                next if copy_exclude.any? { |pattern| pattern.is_a?(Regexp) ? item =~ pattern : File.fnmatch?(pattern, item, File::FNM_DOTMATCH) }
+                if File.directory?(item)
+                  queue += Dir.glob("#{item}/*", File::FNM_DOTMATCH)
+                  FileUtils.mkdir(File.join(destination, item))
+                else
+                  FileUtils.ln(File.join(copy_cache, item), File.join(destination, item))
+                end
+              end
+            end
+          else
             logger.debug "getting (via #{copy_strategy}) revision #{revision} to #{destination}"
             system(command)
+          end
+
           File.open(File.join(destination, "REVISION"), "w") { |f| f.puts(revision) }
 
           logger.trace "compressing #{destination} to #{filename}"
@@ -48,8 +96,24 @@ module Capistrano
           end
         end
 
+        # Returns the location of the local copy cache, if the strategy should
+        # use a local cache + copy instead of a new checkout/export every
+        # time. Returns +nil+ unless :copy_cache has been set. If :copy_cache
+        # is +true+, a default cache location will be returned.
+        def copy_cache
+          @copy_cache ||= configuration[:copy_cache] == true ?
+            File.join(Dir.tmpdir, configuration[:application]) :
+            configuration[:copy_cache]
+        end
+
         private
 
+          # Specify patterns to exclude from the copy. This is only valid
+          # when using a local cache.
+          def copy_exclude
+            @copy_exclude ||= Array(configuration.fetch(:copy_exclude, []))
+          end
+
           # Returns the basename of the release_path, which will be used to
           # name the local copy and archive file.
           def destination

test/deploy/strategy/copy_test.rb

@@ -5,7 +5,8 @@ require 'stringio'
 
 class DeployStrategyCopyTest < Test::Unit::TestCase
   def setup
-    @config = { :logger => Capistrano::Logger.new(:output => StringIO.new),
+    @config = { :application => "captest",
+                :logger => Capistrano::Logger.new(:output => StringIO.new),
                 :releases_path => "/u/apps/test/releases",
                 :release_path => "/u/apps/test/releases/1234567890",
                 :real_revision => "154" }
@@ -16,44 +17,20 @@ class DeployStrategyCopyTest < Test::Unit::TestCase
 
   def test_deploy_with_defaults_should_use_tar_gz_and_checkout
     Dir.expects(:tmpdir).returns("/temp/dir")
-    Dir.expects(:chdir).with("/temp/dir").yields
     @source.expects(:checkout).with("154", "/temp/dir/1234567890").returns(:local_checkout)
-
     @strategy.expects(:system).with(:local_checkout)
-    @strategy.expects(:system).with("tar czf 1234567890.tar.gz 1234567890")
-    @strategy.expects(:put).with(:mock_file_contents, "/tmp/1234567890.tar.gz")
-    @strategy.expects(:run).with("cd /u/apps/test/releases && tar xzf /tmp/1234567890.tar.gz && rm /tmp/1234567890.tar.gz")
-
-    mock_file = mock("file")
-    mock_file.expects(:puts).with("154")
-    File.expects(:open).with("/temp/dir/1234567890/REVISION", "w").yields(mock_file)
-    File.expects(:open).with("/temp/dir/1234567890.tar.gz", "rb").yields(StringIO.new).returns(:mock_file_contents)
-
-    FileUtils.expects(:rm).with("/temp/dir/1234567890.tar.gz")
-    FileUtils.expects(:rm_rf).with("/temp/dir/1234567890")
 
+    prepare_standard_compress_and_copy!
     @strategy.deploy!
   end
 
   def test_deploy_with_export_should_use_tar_gz_and_export
     Dir.expects(:tmpdir).returns("/temp/dir")
-    Dir.expects(:chdir).with("/temp/dir").yields
     @config[:copy_strategy] = :export
     @source.expects(:export).with("154", "/temp/dir/1234567890").returns(:local_export)
-
     @strategy.expects(:system).with(:local_export)
-    @strategy.expects(:system).with("tar czf 1234567890.tar.gz 1234567890")
-    @strategy.expects(:put).with(:mock_file_contents, "/tmp/1234567890.tar.gz")
-    @strategy.expects(:run).with("cd /u/apps/test/releases && tar xzf /tmp/1234567890.tar.gz && rm /tmp/1234567890.tar.gz")
-
-    mock_file = mock("file")
-    mock_file.expects(:puts).with("154")
-    File.expects(:open).with("/temp/dir/1234567890/REVISION", "w").yields(mock_file)
-    File.expects(:open).with("/temp/dir/1234567890.tar.gz", "rb").yields(StringIO.new).returns(:mock_file_contents)
-
-    FileUtils.expects(:rm).with("/temp/dir/1234567890.tar.gz")
-    FileUtils.expects(:rm_rf).with("/temp/dir/1234567890")
 
+    prepare_standard_compress_and_copy!
     @strategy.deploy!
   end
 
@@ -79,7 +56,7 @@ class DeployStrategyCopyTest < Test::Unit::TestCase
     @strategy.deploy!
   end
 
-  def test_deploy_with_bzip2_should_use_zip_and_checkout
+  def test_deploy_with_bzip2_should_use_bz2_and_checkout
     Dir.expects(:tmpdir).returns("/temp/dir")
     Dir.expects(:chdir).with("/temp/dir").yields
     @config[:copy_compression] = :bzip2
@@ -144,4 +121,108 @@ class DeployStrategyCopyTest < Test::Unit::TestCase
 
     @strategy.deploy!
   end
+
+  def test_with_copy_cache_should_checkout_to_cache_if_cache_does_not_exist_and_then_copy
+    @config[:copy_cache] = true
+
+    Dir.stubs(:tmpdir).returns("/temp/dir")
+    File.expects(:exists?).with("/temp/dir/captest").returns(false)
+    Dir.expects(:chdir).with("/temp/dir/captest").yields
+
+    @source.expects(:checkout).with("154", "/temp/dir/captest").returns(:local_checkout)
+    @strategy.expects(:system).with(:local_checkout)
+
+    FileUtils.expects(:mkdir_p).with("/temp/dir/1234567890")
+
+    prepare_directory_tree!("/temp/dir/captest")
+
+    prepare_standard_compress_and_copy!
+    @strategy.deploy!
+  end
+
+  def test_with_copy_cache_should_update_cache_if_cache_exists_and_then_copy
+    @config[:copy_cache] = true
+
+    Dir.stubs(:tmpdir).returns("/temp/dir")
+    File.expects(:exists?).with("/temp/dir/captest").returns(true)
+    Dir.expects(:chdir).with("/temp/dir/captest").yields
+
+    @source.expects(:sync).with("154", "/temp/dir/captest").returns(:local_sync)
+    @strategy.expects(:system).with(:local_sync)
+
+    FileUtils.expects(:mkdir_p).with("/temp/dir/1234567890")
+
+    prepare_directory_tree!("/temp/dir/captest")
+
+    prepare_standard_compress_and_copy!
+    @strategy.deploy!
+  end
+
+  def test_with_copy_cache_with_custom_cache_dir_should_use_specified_cache_dir
+    @config[:copy_cache] = "/u/caches/captest"
+
+    Dir.stubs(:tmpdir).returns("/temp/dir")
+    File.expects(:exists?).with("/u/caches/captest").returns(true)
+    Dir.expects(:chdir).with("/u/caches/captest").yields
+
+    @source.expects(:sync).with("154", "/u/caches/captest").returns(:local_sync)
+    @strategy.expects(:system).with(:local_sync)
+
+    FileUtils.expects(:mkdir_p).with("/temp/dir/1234567890")
+
+    prepare_directory_tree!("/u/caches/captest")
+
+    prepare_standard_compress_and_copy!
+    @strategy.deploy!
+  end
+
+  def test_with_copy_cache_with_excludes_should_not_copy_excluded_files
+    @config[:copy_cache] = true
+    @config[:copy_exclude] = "*/bar.txt"
+
+    Dir.stubs(:tmpdir).returns("/temp/dir")
+    File.expects(:exists?).with("/temp/dir/captest").returns(true)
+    Dir.expects(:chdir).with("/temp/dir/captest").yields
+
+    @source.expects(:sync).with("154", "/temp/dir/captest").returns(:local_sync)
+    @strategy.expects(:system).with(:local_sync)
+
+    FileUtils.expects(:mkdir_p).with("/temp/dir/1234567890")
+
+    prepare_directory_tree!("/temp/dir/captest", true)
+
+    prepare_standard_compress_and_copy!
+    @strategy.deploy!
+  end
+
+  private
+
+    def prepare_directory_tree!(cache, exclude=false)
+      Dir.expects(:glob).with("*", File::FNM_DOTMATCH).returns([".", "..", "app", "foo.txt"])
+      File.expects(:directory?).with("app").returns(true)
+      FileUtils.expects(:mkdir).with("/temp/dir/1234567890/app")
+      File.expects(:directory?).with("foo.txt").returns(false)
+      FileUtils.expects(:ln).with("#{cache}/foo.txt", "/temp/dir/1234567890/foo.txt")
+
+      Dir.expects(:glob).with("app/*", File::FNM_DOTMATCH).returns(["app/.", "app/..", "app/bar.txt"])
+      unless exclude
+        File.expects(:directory?).with("app/bar.txt").returns(false)
+        FileUtils.expects(:ln).with("#{cache}/app/bar.txt", "/temp/dir/1234567890/app/bar.txt")
+      end
+    end
+
+    def prepare_standard_compress_and_copy!
+      Dir.expects(:chdir).with("/temp/dir").yields
+      @strategy.expects(:system).with("tar czf 1234567890.tar.gz 1234567890")
+      @strategy.expects(:put).with(:mock_file_contents, "/tmp/1234567890.tar.gz")
+      @strategy.expects(:run).with("cd /u/apps/test/releases && tar xzf /tmp/1234567890.tar.gz && rm /tmp/1234567890.tar.gz")
+
+      mock_file = mock("file")
+      mock_file.expects(:puts).with("154")
+      File.expects(:open).with("/temp/dir/1234567890/REVISION", "w").yields(mock_file)
+      File.expects(:open).with("/temp/dir/1234567890.tar.gz", "rb").yields(StringIO.new).returns(:mock_file_contents)
+
+      FileUtils.expects(:rm).with("/temp/dir/1234567890.tar.gz")
+      FileUtils.expects(:rm_rf).with("/temp/dir/1234567890")
+    end
 end