From af669cbea386d900ec86510f7d43de6ca4c1771e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Mon, 17 Oct 2016 18:52:14 +0200 Subject: [PATCH 1/4] Change the approach to check if patches apply cleanly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- .gitlab-ci.yml | 2 +- lib/gitlab/ee_compat_check.rb | 260 ++++++++++++++++++++++++++++ lib/tasks/ce_to_ee_merge_check.rake | 4 - lib/tasks/ee_compat_check.rake | 4 + lib/tasks/gitlab/dev.rake | 107 ++---------- 5 files changed, 276 insertions(+), 101 deletions(-) create mode 100644 lib/gitlab/ee_compat_check.rb delete mode 100644 lib/tasks/ce_to_ee_merge_check.rake create mode 100644 lib/tasks/ee_compat_check.rake diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 76117a48730..9c4b4acbaf5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -210,7 +210,7 @@ rake brakeman: *exec rake flay: *exec license_finder: *exec rake downtime_check: *exec -rake ce_to_ee_merge_check: +rake ee_compat_check: <<: *exec only: - branches diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb new file mode 100644 index 00000000000..5e8a0c14eba --- /dev/null +++ b/lib/gitlab/ee_compat_check.rb @@ -0,0 +1,260 @@ +module Gitlab + # Checks if a set of migrations requires downtime or not. + class EeCompatCheck + EE_REPO = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze + + attr_reader :ce_branch, :check_dir, :ce_repo + + def initialize(branch:, check_dir:, ce_repo: nil) + @ce_branch = branch + @check_dir = check_dir + @ce_repo = ce_repo || 'https://gitlab.com/gitlab-org/gitlab-ce.git' + end + + def check + ensure_ee_repo + delete_patches + + generate_patch(ce_branch, ce_patch_full_path) + + Dir.chdir(check_dir) do + step("In the #{check_dir} directory") + + step("Pulling latest master", %w[git pull --ff-only origin master]) + + status = catch(:halt_check) do + ce_branch_compat_check! + + delete_ee_branch_locally + + ee_branch_presence_check! + + ee_branch_compat_check! + end + + delete_ee_branch_locally + delete_patches + + if status.nil? + true + else + false + end + end + end + + private + + def ensure_ee_repo + if Dir.exist?(check_dir) + step("#{check_dir} already exists") + else + cmd = %W[git clone --branch master --single-branch --depth 1 #{EE_REPO} #{check_dir}] + step("Cloning #{EE_REPO} into #{check_dir}", cmd) + end + end + + def ce_branch_compat_check! + cmd = %W[git apply --check #{ce_patch_full_path}] + status = step("Checking if #{ce_patch_name} applies cleanly to EE/master", cmd) + + if status.zero? + puts ce_applies_cleanly_msg(ce_branch) + throw(:halt_check) + end + end + + def ee_branch_presence_check! + status = step("Fetching origin/#{ee_branch}", %W[git fetch origin #{ee_branch}]) + + unless status.zero? + puts + puts ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg + + throw(:halt_check, :ko) + end + end + + def ee_branch_compat_check! + step("Checking out origin/#{ee_branch}", %W[git checkout -b #{ee_branch} FETCH_HEAD]) + + generate_patch(ee_branch, ee_patch_full_path) + cmd = %W[git apply --check #{ee_patch_full_path}] + status = step("Checking if #{ee_patch_name} applies cleanly to EE/master", cmd) + + unless status.zero? + puts + puts ee_branch_doesnt_apply_cleanly_msg + + throw(:halt_check, :ko) + end + + puts + puts ee_applies_cleanly_msg + end + + def generate_patch(branch, filepath) + FileUtils.rm(filepath, force: true) + + depth = 0 + loop do + depth += 10 + step("Fetching origin/master", %W[git fetch origin master --depth=#{depth}]) + status = step("Finding merge base with master", %W[git merge-base FETCH_HEAD #{branch}]) + + break if status.zero? || depth > 500 + end + + raise "#{branch} is too far behind master, please rebase it!" if depth > 500 + + step("Generating the patch against master") + output, status = Gitlab::Popen.popen(%w[git format-patch FETCH_HEAD --stdout]) + throw(:halt_check, :ko) unless status.zero? + + File.open(filepath, 'w+') { |f| f.write(output) } + throw(:halt_check, :ko) unless File.exist?(filepath) + end + + def delete_ee_branch_locally + step("Checking out origin/master", %w[git checkout master]) + step("Deleting the local #{ee_branch} branch", %W[git branch -D #{ee_branch}]) + end + + def delete_patches + step("Deleting #{ce_patch_full_path}") + FileUtils.rm(ce_patch_full_path, force: true) + + step("Deleting #{ee_patch_full_path}") + FileUtils.rm(ee_patch_full_path, force: true) + end + + def ce_patch_name + @ce_patch_name ||= "#{ce_branch}.patch" + end + + def ce_patch_full_path + @ce_patch_full_path ||= File.expand_path(ce_patch_name, check_dir) + end + + def ee_branch + @ee_branch ||= "#{ce_branch}-ee" + end + + def ee_patch_name + @ee_patch_name ||= "#{ee_branch}.patch" + end + + def ee_patch_full_path + @ee_patch_full_path ||= File.expand_path(ee_patch_name, check_dir) + end + + def step(desc, cmd = nil) + puts "\n=> #{desc}\n" + + if cmd + puts "\n$ #{cmd.join(' ')}" + command(cmd) + end + end + + def command(cmd) + output, status = Gitlab::Popen.popen(cmd) + puts output + + status + end + + def ce_applies_cleanly_msg(ce_branch) + <<-MSG.strip_heredoc + ================================================================= + 🎉 Congratulations!! 🎉 + + The #{ce_branch} branch applies cleanly to EE/master! + + Much ❤️!! + =================================================================\n + MSG + end + + def ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg + <<-MSG.strip_heredoc + ================================================================= + 💥 Oh no! 💥 + + The #{ce_branch} branch does not apply cleanly to the current + EE/master, and no #{ee_branch} branch was found in the EE repository. + + Please create a #{ee_branch} branch that includes changes from + #{ce_branch} but also specific changes than can be applied cleanly + to EE/master. + + There are different ways to create such branch: + + 1. Create a new branch based on the CE branch and rebase it on top of EE/master + + # In the EE repo + $ git fetch #{ce_repo} #{ce_branch} + $ git checkout -b #{ee_branch} FETCH_HEAD + + # You can squash the #{ce_branch} commits into a single "Port of #{ce_branch} to EE" commit + # before rebasing to limit the conflicts-resolving steps during the rebase + $ git fetch origin + $ git rebase origin/master + + At this point you will likely have conflicts. + Solve them, and continue/finish the rebase. + + You can squash the #{ce_branch} commits into a single "Port of #{ce_branch} to EE". + + 2. Create a new branch from master and cherry-pick your CE commits + + # In the EE repo + $ git fetch origin + $ git checkout -b #{ee_branch} FETCH_HEAD + $ git fetch #{ce_repo} #{ce_branch} + $ git cherry-pick SHA # Repeat for all the commits you want to pick + + You can squash the #{ce_branch} commits into a single "Port of #{ce_branch} to EE" commit. + + Don't forget to push your branch to #{EE_REPO}: + + # In the EE repo + $ git push origin #{ee_branch} + + You can then retry this failed build, and hopefully it should pass. + + Stay 💪 ! + =================================================================\n + MSG + end + + def ee_branch_doesnt_apply_cleanly_msg + <<-MSG.strip_heredoc + ================================================================= + 💥 Oh no! 💥 + + The #{ce_branch} does not apply cleanly to the current + EE/master, and even though a #{ee_branch} branch exists in the EE + repository, it does not apply cleanly either to EE/master! + + Please update the #{ee_branch}, push it again to #{EE_REPO}, and + retry this build. + + Stay 💪 ! + =================================================================\n + MSG + end + + def ee_applies_cleanly_msg + <<-MSG.strip_heredoc + ================================================================= + 🎉 Congratulations!! 🎉 + + The #{ee_branch} branch applies cleanly to EE/master! + + Much ❤️!! + =================================================================\n + MSG + end + end +end diff --git a/lib/tasks/ce_to_ee_merge_check.rake b/lib/tasks/ce_to_ee_merge_check.rake deleted file mode 100644 index 424e7883060..00000000000 --- a/lib/tasks/ce_to_ee_merge_check.rake +++ /dev/null @@ -1,4 +0,0 @@ -desc 'Checks if the branch would apply cleanly to EE' -task ce_to_ee_merge_check: :environment do - Rake::Task['gitlab:dev:ce_to_ee_merge_check'].invoke -end diff --git a/lib/tasks/ee_compat_check.rake b/lib/tasks/ee_compat_check.rake new file mode 100644 index 00000000000..f494fa5c5c2 --- /dev/null +++ b/lib/tasks/ee_compat_check.rake @@ -0,0 +1,4 @@ +desc 'Checks if the branch would apply cleanly to EE' +task ee_compat_check: :environment do + Rake::Task['gitlab:dev:ee_compat_check'].invoke +end diff --git a/lib/tasks/gitlab/dev.rake b/lib/tasks/gitlab/dev.rake index 47bdb2d32d2..5ee99dfc810 100644 --- a/lib/tasks/gitlab/dev.rake +++ b/lib/tasks/gitlab/dev.rake @@ -1,106 +1,21 @@ namespace :gitlab do namespace :dev do desc 'Checks if the branch would apply cleanly to EE' - task ce_to_ee_merge_check: :environment do + task ee_compat_check: :environment do return if defined?(Gitlab::License) return unless ENV['CI'] - ce_repo = ENV['CI_BUILD_REPO'] - ce_branch = ENV['CI_BUILD_REF_NAME'] + success = + Gitlab::EeCompatCheck.new( + branch: ENV['CI_BUILD_REF_NAME'], + check_dir: File.expand_path('ee-compat-check', __dir__), + ce_repo: ENV['CI_BUILD_REPO'] + ).check - ee_repo = 'https://gitlab.com/gitlab-org/gitlab-ee.git' - ee_branch = "#{ce_branch}-ee" - ee_dir = 'gitlab-ee-merge-check' - - puts "\n=> Cloning #{ee_repo} into #{ee_dir}\n" - `git clone #{ee_repo} #{ee_dir} --depth 1` - Dir.chdir(ee_dir) do - puts "\n => Fetching #{ce_repo}/#{ce_branch}\n" - `git fetch #{ce_repo} #{ce_branch} --depth 1` - - # Try to merge the current tested branch to EE/master... - puts "\n => Merging #{ce_repo}/#{ce_branch} into #{ee_repo}/master\n" - `git merge FETCH_HEAD` - - exit 0 if $?.success? - - # Check if the -ee branch exists... - puts "\n => Check if #{ee_repo}/#{ee_branch} exists\n" - `git rev-parse --verify #{ee_branch}` - - # The -ee doesn't exist - unless $?.success? - puts - puts <<-MSG.strip_heredoc - ================================================================= - The #{ce_branch} branch cannot be merged without conflicts to the - current EE/master, and no #{ee_branch} branch was detected in - the EE repository. - - Please create a #{ee_branch} branch that includes changes from - #{ce_branch} but also specific changes than can be applied cleanly - to EE/master. - - You can create this branch as follows: - - 1. In the EE repo: - $ git fetch origin - $ git fetch #{ce_repo} #{ce_branch} - $ git checkout -b #{ee_branch} FETCH_HEAD - $ git rebase origin/master - 2. At this point you will likely have conflicts, solve them, and - continue/finish the rebase. Note: You can squash the CE commits - before rebasing. - 3. You can squash all the original #{ce_branch} commits into a - single "Port of #{ce_branch} to EE". - 4. Push your branch to #{ee_repo}: - $ git push origin #{ee_branch} - =================================================================\n - MSG - - exit 1 - end - - # Try to merge the -ee branch to EE/master... - puts "\n => Merging #{ee_repo}/#{ee_branch} into #{ee_repo}/master\n" - `git merge #{ee_branch} master` - - # The -ee cannot be merged cleanly to EE/master... - unless $?.success? - puts - puts <<-MSG.strip_heredoc - ================================================================= - The #{ce_branch} branch cannot be merged without conflicts to - EE/master, and even though the #{ee_branch} branch exists in the EE - repository, it cannot be merged without conflicts to EE/master. - - Please update the #{ee_branch}, push it again to #{ee_repo}, and - retry this job. - =================================================================\n - MSG - - exit 2 - end - - puts "\n => Merging #{ce_repo}/#{ce_branch} into #{ee_repo}/master\n" - `git merge FETCH_HEAD` - exit 0 if $?.success? - - # The -ee can be merged cleanly to EE/master, but still - # cannot be merged cleanly to EE/master... - puts - puts <<-MSG.strip_heredoc - ================================================================= - The #{ce_branch} branch cannot be merged without conflicts to EE, and - even though the #{ee_branch} branch exists in the EE repository and - applies cleanly to EE/master, it doesn't prevent conflicts when - merging #{ce_branch} into EE. - - We may be in a complex situation here. - =================================================================\n - MSG - - exit 3 + if success + exit 0 + else + exit 1 end end end From c7af4cf83027863a2d109ba82135e56092c71a09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Mon, 24 Oct 2016 20:29:00 +0200 Subject: [PATCH 2/4] Don't print out implementation detail step MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- lib/gitlab/ee_compat_check.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index 5e8a0c14eba..6f8d2ca3220 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -116,7 +116,7 @@ module Gitlab end def delete_ee_branch_locally - step("Checking out origin/master", %w[git checkout master]) + command(%w[git checkout master]) step("Deleting the local #{ee_branch} branch", %W[git branch -D #{ee_branch}]) end From 402dcab4f1a9e2ea7310e9d6137abe1299f48bf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Mon, 24 Oct 2016 20:47:50 +0200 Subject: [PATCH 3/4] Use File.write instead of File.open + File#write MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- lib/gitlab/ee_compat_check.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index 6f8d2ca3220..fd9fd56510f 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -111,7 +111,7 @@ module Gitlab output, status = Gitlab::Popen.popen(%w[git format-patch FETCH_HEAD --stdout]) throw(:halt_check, :ko) unless status.zero? - File.open(filepath, 'w+') { |f| f.write(output) } + File.write(filepath, output) throw(:halt_check, :ko) unless File.exist?(filepath) end From d38d4a9e8e4b8aff1419f146c40bf7f4af8669ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Mon, 24 Oct 2016 21:19:46 +0200 Subject: [PATCH 4/4] Disable Rails/Output cop since it makes no sense here MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- lib/gitlab/ee_compat_check.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index fd9fd56510f..b1a6d5fe0f6 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -1,3 +1,4 @@ +# rubocop: disable Rails/Output module Gitlab # Checks if a set of migrations requires downtime or not. class EeCompatCheck