From bf89d31d395cdbb43303ccc0a876d1be731efb7d Mon Sep 17 00:00:00 2001 From: Jamis Buck Date: Sat, 8 Mar 2008 19:07:46 +0000 Subject: [PATCH] Improved "copy" strategy, including support for local caching and pattern exclusion git-svn-id: http://svn.rubyonrails.org/rails/tools/capistrano@8993 5ecf4fe2-1ee6-0310-87b1-e25e094e27de --- CHANGELOG | 5 + .../recipes/deploy/strategy/base.rb | 6 + .../recipes/deploy/strategy/copy.rb | 70 ++++++++- test/deploy/strategy/copy_test.rb | 137 ++++++++++++++---- 4 files changed, 187 insertions(+), 31 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 78fa16a2..7e8a102a 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,8 @@ +*SVN* + +* Improved "copy" strategy supports local caching and pattern exclusion (via :copy_cache and :copy_exclude variables) [Jamis Buck] + + *2.2.0* February 27, 2008 * Fix git submodule support to init on sync [halorgium] diff --git a/lib/capistrano/recipes/deploy/strategy/base.rb b/lib/capistrano/recipes/deploy/strategy/base.rb index 847711b2..2884a73a 100644 --- a/lib/capistrano/recipes/deploy/strategy/base.rb +++ b/lib/capistrano/recipes/deploy/strategy/base.rb @@ -46,6 +46,12 @@ module Capistrano end end + # A wrapper for Kernel#system that logs the command being executed. + def system(*args) + logger.trace "executing locally: #{args.join(' ')}" + super + end + private def logger diff --git a/lib/capistrano/recipes/deploy/strategy/copy.rb b/lib/capistrano/recipes/deploy/strategy/copy.rb index 9fc14e9d..2f24d7d9 100644 --- a/lib/capistrano/recipes/deploy/strategy/copy.rb +++ b/lib/capistrano/recipes/deploy/strategy/copy.rb @@ -15,7 +15,26 @@ module Capistrano # of the source code. If you would rather use the export operation, # you can set the :copy_strategy variable to :export. # - # This deployment strategy supports a special variable, + # set :copy_strategy, :export + # + # For even faster deployments, you can set the :copy_cache variable to + # true. This will cause deployments to do a new checkout of your + # repository to a new directory, and then copy that checkout. Subsequent + # deploys will just resync that copy, rather than doing an entirely new + # checkout. Additionally, you can specify file patterns to exclude from + # the copy when using :copy_cache; just set the :copy_exclude variable + # to an array of file globs or regexps. + # + # set :copy_cache, true + # set :copy_exclude, ".git/*" + # + # Note that :copy_strategy is ignored when :copy_cache is set. Also, if + # you want the copy cache put somewhere specific, you can set the variable + # to the path you want, instead of merely 'true': + # + # set :copy_cache, "/tmp/caches/myapp" + # + # This deployment strategy also supports a special variable, # :copy_compression, which must be one of :gzip, :bz2, or # :zip, and which specifies how the source should be compressed for # transmission to each host. @@ -25,8 +44,37 @@ module Capistrano # servers, and uncompresses it on each of them into the deployment # directory. def deploy! - logger.debug "getting (via #{copy_strategy}) revision #{revision} to #{destination}" - system(command) + if copy_cache + if File.exists?(copy_cache) + logger.debug "refreshing local cache to revision #{revision} at #{copy_cache}" + system(source.sync(revision, copy_cache)) + else + logger.debug "preparing local cache at #{copy_cache}" + system(source.checkout(revision, copy_cache)) + end + + logger.debug "copying cache to deployment staging area #{destination}" + Dir.chdir(copy_cache) do + FileUtils.mkdir_p(destination) + queue = Dir.glob("*", File::FNM_DOTMATCH) + while queue.any? + item = queue.shift + name = File.basename(item) + next if name == "." || name == ".." + next if copy_exclude.any? { |pattern| pattern.is_a?(Regexp) ? item =~ pattern : File.fnmatch?(pattern, item, File::FNM_DOTMATCH) } + if File.directory?(item) + queue += Dir.glob("#{item}/*", File::FNM_DOTMATCH) + FileUtils.mkdir(File.join(destination, item)) + else + FileUtils.ln(File.join(copy_cache, item), File.join(destination, item)) + end + end + end + else + logger.debug "getting (via #{copy_strategy}) revision #{revision} to #{destination}" + system(command) + end + File.open(File.join(destination, "REVISION"), "w") { |f| f.puts(revision) } logger.trace "compressing #{destination} to #{filename}" @@ -48,8 +96,24 @@ module Capistrano end end + # Returns the location of the local copy cache, if the strategy should + # use a local cache + copy instead of a new checkout/export every + # time. Returns +nil+ unless :copy_cache has been set. If :copy_cache + # is +true+, a default cache location will be returned. + def copy_cache + @copy_cache ||= configuration[:copy_cache] == true ? + File.join(Dir.tmpdir, configuration[:application]) : + configuration[:copy_cache] + end + private + # Specify patterns to exclude from the copy. This is only valid + # when using a local cache. + def copy_exclude + @copy_exclude ||= Array(configuration.fetch(:copy_exclude, [])) + end + # Returns the basename of the release_path, which will be used to # name the local copy and archive file. def destination diff --git a/test/deploy/strategy/copy_test.rb b/test/deploy/strategy/copy_test.rb index b53ec8aa..d37dea11 100644 --- a/test/deploy/strategy/copy_test.rb +++ b/test/deploy/strategy/copy_test.rb @@ -5,7 +5,8 @@ require 'stringio' class DeployStrategyCopyTest < Test::Unit::TestCase def setup - @config = { :logger => Capistrano::Logger.new(:output => StringIO.new), + @config = { :application => "captest", + :logger => Capistrano::Logger.new(:output => StringIO.new), :releases_path => "/u/apps/test/releases", :release_path => "/u/apps/test/releases/1234567890", :real_revision => "154" } @@ -16,44 +17,20 @@ class DeployStrategyCopyTest < Test::Unit::TestCase def test_deploy_with_defaults_should_use_tar_gz_and_checkout Dir.expects(:tmpdir).returns("/temp/dir") - Dir.expects(:chdir).with("/temp/dir").yields @source.expects(:checkout).with("154", "/temp/dir/1234567890").returns(:local_checkout) - @strategy.expects(:system).with(:local_checkout) - @strategy.expects(:system).with("tar czf 1234567890.tar.gz 1234567890") - @strategy.expects(:put).with(:mock_file_contents, "/tmp/1234567890.tar.gz") - @strategy.expects(:run).with("cd /u/apps/test/releases && tar xzf /tmp/1234567890.tar.gz && rm /tmp/1234567890.tar.gz") - - mock_file = mock("file") - mock_file.expects(:puts).with("154") - File.expects(:open).with("/temp/dir/1234567890/REVISION", "w").yields(mock_file) - File.expects(:open).with("/temp/dir/1234567890.tar.gz", "rb").yields(StringIO.new).returns(:mock_file_contents) - - FileUtils.expects(:rm).with("/temp/dir/1234567890.tar.gz") - FileUtils.expects(:rm_rf).with("/temp/dir/1234567890") + prepare_standard_compress_and_copy! @strategy.deploy! end def test_deploy_with_export_should_use_tar_gz_and_export Dir.expects(:tmpdir).returns("/temp/dir") - Dir.expects(:chdir).with("/temp/dir").yields @config[:copy_strategy] = :export @source.expects(:export).with("154", "/temp/dir/1234567890").returns(:local_export) - @strategy.expects(:system).with(:local_export) - @strategy.expects(:system).with("tar czf 1234567890.tar.gz 1234567890") - @strategy.expects(:put).with(:mock_file_contents, "/tmp/1234567890.tar.gz") - @strategy.expects(:run).with("cd /u/apps/test/releases && tar xzf /tmp/1234567890.tar.gz && rm /tmp/1234567890.tar.gz") - - mock_file = mock("file") - mock_file.expects(:puts).with("154") - File.expects(:open).with("/temp/dir/1234567890/REVISION", "w").yields(mock_file) - File.expects(:open).with("/temp/dir/1234567890.tar.gz", "rb").yields(StringIO.new).returns(:mock_file_contents) - - FileUtils.expects(:rm).with("/temp/dir/1234567890.tar.gz") - FileUtils.expects(:rm_rf).with("/temp/dir/1234567890") + prepare_standard_compress_and_copy! @strategy.deploy! end @@ -79,7 +56,7 @@ class DeployStrategyCopyTest < Test::Unit::TestCase @strategy.deploy! end - def test_deploy_with_bzip2_should_use_zip_and_checkout + def test_deploy_with_bzip2_should_use_bz2_and_checkout Dir.expects(:tmpdir).returns("/temp/dir") Dir.expects(:chdir).with("/temp/dir").yields @config[:copy_compression] = :bzip2 @@ -144,4 +121,108 @@ class DeployStrategyCopyTest < Test::Unit::TestCase @strategy.deploy! end + + def test_with_copy_cache_should_checkout_to_cache_if_cache_does_not_exist_and_then_copy + @config[:copy_cache] = true + + Dir.stubs(:tmpdir).returns("/temp/dir") + File.expects(:exists?).with("/temp/dir/captest").returns(false) + Dir.expects(:chdir).with("/temp/dir/captest").yields + + @source.expects(:checkout).with("154", "/temp/dir/captest").returns(:local_checkout) + @strategy.expects(:system).with(:local_checkout) + + FileUtils.expects(:mkdir_p).with("/temp/dir/1234567890") + + prepare_directory_tree!("/temp/dir/captest") + + prepare_standard_compress_and_copy! + @strategy.deploy! + end + + def test_with_copy_cache_should_update_cache_if_cache_exists_and_then_copy + @config[:copy_cache] = true + + Dir.stubs(:tmpdir).returns("/temp/dir") + File.expects(:exists?).with("/temp/dir/captest").returns(true) + Dir.expects(:chdir).with("/temp/dir/captest").yields + + @source.expects(:sync).with("154", "/temp/dir/captest").returns(:local_sync) + @strategy.expects(:system).with(:local_sync) + + FileUtils.expects(:mkdir_p).with("/temp/dir/1234567890") + + prepare_directory_tree!("/temp/dir/captest") + + prepare_standard_compress_and_copy! + @strategy.deploy! + end + + def test_with_copy_cache_with_custom_cache_dir_should_use_specified_cache_dir + @config[:copy_cache] = "/u/caches/captest" + + Dir.stubs(:tmpdir).returns("/temp/dir") + File.expects(:exists?).with("/u/caches/captest").returns(true) + Dir.expects(:chdir).with("/u/caches/captest").yields + + @source.expects(:sync).with("154", "/u/caches/captest").returns(:local_sync) + @strategy.expects(:system).with(:local_sync) + + FileUtils.expects(:mkdir_p).with("/temp/dir/1234567890") + + prepare_directory_tree!("/u/caches/captest") + + prepare_standard_compress_and_copy! + @strategy.deploy! + end + + def test_with_copy_cache_with_excludes_should_not_copy_excluded_files + @config[:copy_cache] = true + @config[:copy_exclude] = "*/bar.txt" + + Dir.stubs(:tmpdir).returns("/temp/dir") + File.expects(:exists?).with("/temp/dir/captest").returns(true) + Dir.expects(:chdir).with("/temp/dir/captest").yields + + @source.expects(:sync).with("154", "/temp/dir/captest").returns(:local_sync) + @strategy.expects(:system).with(:local_sync) + + FileUtils.expects(:mkdir_p).with("/temp/dir/1234567890") + + prepare_directory_tree!("/temp/dir/captest", true) + + prepare_standard_compress_and_copy! + @strategy.deploy! + end + + private + + def prepare_directory_tree!(cache, exclude=false) + Dir.expects(:glob).with("*", File::FNM_DOTMATCH).returns([".", "..", "app", "foo.txt"]) + File.expects(:directory?).with("app").returns(true) + FileUtils.expects(:mkdir).with("/temp/dir/1234567890/app") + File.expects(:directory?).with("foo.txt").returns(false) + FileUtils.expects(:ln).with("#{cache}/foo.txt", "/temp/dir/1234567890/foo.txt") + + Dir.expects(:glob).with("app/*", File::FNM_DOTMATCH).returns(["app/.", "app/..", "app/bar.txt"]) + unless exclude + File.expects(:directory?).with("app/bar.txt").returns(false) + FileUtils.expects(:ln).with("#{cache}/app/bar.txt", "/temp/dir/1234567890/app/bar.txt") + end + end + + def prepare_standard_compress_and_copy! + Dir.expects(:chdir).with("/temp/dir").yields + @strategy.expects(:system).with("tar czf 1234567890.tar.gz 1234567890") + @strategy.expects(:put).with(:mock_file_contents, "/tmp/1234567890.tar.gz") + @strategy.expects(:run).with("cd /u/apps/test/releases && tar xzf /tmp/1234567890.tar.gz && rm /tmp/1234567890.tar.gz") + + mock_file = mock("file") + mock_file.expects(:puts).with("154") + File.expects(:open).with("/temp/dir/1234567890/REVISION", "w").yields(mock_file) + File.expects(:open).with("/temp/dir/1234567890.tar.gz", "rb").yields(StringIO.new).returns(:mock_file_contents) + + FileUtils.expects(:rm).with("/temp/dir/1234567890.tar.gz") + FileUtils.expects(:rm_rf).with("/temp/dir/1234567890") + end end