From ab2571735c894ba87635961f4699fd2e5d32eeec Mon Sep 17 00:00:00 2001 From: Sanad Liaquat Date: Tue, 18 Jun 2019 13:29:30 +0500 Subject: [PATCH] Add script for MR with many commits Also does a lot of cleanup and refactoring --- qa/Rakefile | 5 +- qa/qa/tools/generate_perf_testdata.rb | 185 ++++++++++++++++++-------- 2 files changed, 132 insertions(+), 58 deletions(-) diff --git a/qa/Rakefile b/qa/Rakefile index 7ac018f7286..7ba8a6d68ba 100644 --- a/qa/Rakefile +++ b/qa/Rakefile @@ -13,8 +13,9 @@ task :delete_subgroups do end desc "Generate Performance Testdata" -task :generate_perf_testdata do - QA::Tools::GeneratePerfTestdata.new.run +task :generate_perf_testdata, :type do |t, args| + args.with_defaults(type: :all) + QA::Tools::GeneratePerfTestdata.new.method(args[:type]).call end desc "Run artillery load tests" diff --git a/qa/qa/tools/generate_perf_testdata.rb b/qa/qa/tools/generate_perf_testdata.rb index de8cfa1aed9..b0477951967 100644 --- a/qa/qa/tools/generate_perf_testdata.rb +++ b/qa/qa/tools/generate_perf_testdata.rb @@ -19,26 +19,30 @@ module QA raise ArgumentError, "Please provide GITLAB_QA_ACCESS_TOKEN" unless ENV['GITLAB_QA_ACCESS_TOKEN'] @api_client = Runtime::API::Client.new(ENV['GITLAB_ADDRESS'], personal_access_token: ENV['GITLAB_QA_ACCESS_TOKEN']) - @group_name = "gitlab-qa-perf-sandbox-#{SecureRandom.hex(8)}" - @project_name = "my-test-project-#{SecureRandom.hex(8)}" + @group_name = ENV['GROUP_NAME'] || "gitlab-qa-perf-sandbox-#{SecureRandom.hex(8)}" + @project_name = ENV['PROJECT_NAME'] || "my-test-project-#{SecureRandom.hex(8)}" @visibility = "public" @urls = { host: ENV['GITLAB_ADDRESS'] } end - def run + def all STDOUT.puts 'Running...' group_id = create_group create_project(group_id) - create_branch - add_new_file + + create_many_branches + create_many_new_files + create_mr_with_many_commits + methods_arr = [ - method(:create_issues), - method(:create_labels), - method(:create_todos), - method(:create_merge_requests), - method(:create_issue_with_500_discussions), - method(:create_mr_with_large_files) + method(:create_many_issues), + method(:create_many_labels), + method(:create_many_todos), + method(:create_many_merge_requests), + method(:create_an_issue_with_many_discussions), + method(:create_an_mr_with_large_files_and_many_mr_discussions) ] + threads_arr = [] methods_arr.each do |m| @@ -51,103 +55,102 @@ module QA STDOUT.puts "\nDone" end - private - def create_group - group_search_response = post Runtime::API::Request.new(@api_client, "/groups").url, "name=#{@group_name}&path=#{@group_name}&visibility=#{@visibility}" + group_search_response = create_a_group_api_req(@group_name, @visibility) group = JSON.parse(group_search_response.body) @urls[:group_page] = group["web_url"] group["id"] + STDOUT.puts "Created a group: #{@urls[:group_page]}" end def create_project(group_id) - create_project_response = post Runtime::API::Request.new(@api_client, "/projects").url, "name=#{@project_name}&namespace_id=#{group_id}&visibility=#{@visibility}" + create_project_response = create_a_project_api_req(@project_name, group_id, @visibility) @urls[:project_page] = JSON.parse(create_project_response.body)["web_url"] + STDOUT.puts "Created a project: #{@urls[:project_page]}" end - def create_issues + def create_many_issues 30.times do |i| - post Runtime::API::Request.new(@api_client, "/projects/#{@group_name}%2F#{@project_name}/issues").url, "title=issue#{i}&description=desc#{i}" + create_an_issue_api_req("#{@group_name}%2F#{@project_name}", "issue#{i}", "desc#{i}") end @urls[:issues_list_page] = @urls[:project_page] + "/issues" - STDOUT.puts "Created Issues" + STDOUT.puts "Created many issues: #{@urls[:issues_list_page]}" end - def create_todos + def create_many_todos 30.times do |i| - post Runtime::API::Request.new(@api_client, "/projects/#{@group_name}%2F#{@project_name}/issues/#{i + 1}/todo").url, nil + create_a_todo_api_req("#{@group_name}%2F#{@project_name}", "#{i + 1}") end @urls[:todos_page] = ENV['GITLAB_ADDRESS'] + "/dashboard/todos" - STDOUT.puts "Created todos" + STDOUT.puts "Created many todos: #{@urls[:todos_page]}" end - def create_labels + def create_many_labels 30.times do |i| - post Runtime::API::Request.new(@api_client, "/projects/#{@group_name}%2F#{@project_name}/labels").url, - "name=label#{i}&color=#{Faker::Color.hex_color}" + create_a_label_api_req("#{@group_name}%2F#{@project_name}", "label#{i}", "#{Faker::Color.hex_color}") end @urls[:labels_page] = @urls[:project_page] + "/labels" - STDOUT.puts "Created labels" + STDOUT.puts "Created many labels: #{@urls[:labels_page]}" end - def create_merge_requests + def create_many_merge_requests 30.times do |i| - post Runtime::API::Request.new(@api_client, "/projects/#{@group_name}%2F#{@project_name}/merge_requests").url, "source_branch=branch#{i}&target_branch=master&title=MR#{i}" + create_a_merge_request_api_req("#{@group_name}%2F#{@project_name}", "branch#{i}", "master", "MR#{i}") end @urls[:mr_list_page] = @urls[:project_page] + "/merge_requests" - STDOUT.puts "Created MRs" + STDOUT.puts "Created many MRs: #{@urls[:mr_list_page]}" end - def add_new_file - post Runtime::API::Request.new(@api_client, "/projects/#{@group_name}%2F#{@project_name}/repository/files/hello.txt").url, "branch=master&commit_message=\"hello\"&content=\"my new content\"" + def create_many_new_files + create_a_new_file_api_req("hello.txt", "master", "#{@group_name}%2F#{@project_name}", "hello", "my new content") 30.times do |i| - post Runtime::API::Request.new(@api_client, "/projects/#{@group_name}%2F#{@project_name}/repository/files/hello#{i}.txt").url, "branch=branch#{i}&commit_message=\"hello\"&content=\"my new content\"" + create_a_new_file_api_req("hello#{i}.txt", "branch#{i}", "#{@group_name}%2F#{@project_name}", "hello", "my new content") end - STDOUT.puts "Added Files" + + @urls[:files_page] = @urls[:project_page] + "/tree/master" + STDOUT.puts "Added many new files: #{@urls[:files_page]}" end - def create_branch + def create_many_branches 30.times do |i| - post Runtime::API::Request.new(@api_client, "/projects/#{@group_name}%2F#{@project_name}/repository/branches").url, "branch=branch#{i}&ref=master" + create_a_branch_api_req("branch#{i}", "#{@group_name}%2F#{@project_name}") end - STDOUT.puts "Created branches" + @urls[:branches_page] = @urls[:project_page] + "/-/branches" + STDOUT.puts "Created many branches: #{@urls[:branches_page]}" end - def create_issue_with_500_discussions + def create_an_issue_with_many_discussions issue_id = 1 500.times do - post Runtime::API::Request.new(@api_client, "/projects/#{@group_name}%2F#{@project_name}/issues/#{issue_id}/discussions").url, "body=\"Let us discuss\"" + create_a_discussion_on_issue_api_req("#{@group_name}%2F#{@project_name}", issue_id, "Let us discuss") end - labels_list = (0..15).map {|i| "label#{i}"}.join(',') + labels_list = (0..15).map { |i| "label#{i}" }.join(',') # Add description and labels - put Runtime::API::Request.new(@api_client, "/projects/#{@group_name}%2F#{@project_name}/issues/#{issue_id}").url, "description=#{Faker::Lorem.sentences(500).join(" ")}&labels=#{labels_list}" + update_an_issue_api_req("#{@group_name}%2F#{@project_name}", issue_id, "#{Faker::Lorem.sentences(500).join(" ")}", labels_list) @urls[:large_issue] = @urls[:project_page] + "/issues/#{issue_id}" - STDOUT.puts "Created Issue with 500 Discussions" + STDOUT.puts "Created an issue with many discussions: #{@urls[:large_issue]}" end - def create_mr_with_large_files + def create_an_mr_with_large_files_and_many_mr_discussions content_arr = [] 16.times do |i| faker_line_arr = Faker::Lorem.sentences(1500) content = faker_line_arr.join("\n\r") - post Runtime::API::Request.new(@api_client, "/projects/#{@group_name}%2F#{@project_name}/repository/files/hello#{i}.txt").url, - "branch=master&commit_message=\"Add hello#{i}.txt\"&content=#{content}" + create_a_new_file_api_req("hello#{i}.txt", "master", "#{@group_name}%2F#{@project_name}", "Add hello#{i}.txt", content) content_arr[i] = faker_line_arr end - post Runtime::API::Request.new(@api_client, "/projects/#{@group_name}%2F#{@project_name}/repository/branches").url, - "branch=performance&ref=master" + create_a_branch_api_req("performance", "#{@group_name}%2F#{@project_name}") 16.times do |i| missed_line_array = content_arr[i].each_slice(2).map(&:first) content = missed_line_array.join("\n\rIm new!:D \n\r ") - put Runtime::API::Request.new(@api_client, "/projects/#{@group_name}%2F#{@project_name}/repository/files/hello#{i}.txt").url, - "branch=performance&commit_message=\"Update hello#{i}.txt\"&content=#{content}" + + update_file_api_req("hello#{i}.txt", "performance", "#{@group_name}%2F#{@project_name}", "Update hello#{i}.txt", content) end - create_mr_response = post Runtime::API::Request.new(@api_client, """/projects/#{@group_name}%2F#{@project_name}/merge_requests""").url, - "source_branch=performance&target_branch=master&title=Large_MR" + create_mr_response = create_a_merge_request_api_req("#{@group_name}%2F#{@project_name}", "performance", "master", "Large_MR") iid = JSON.parse(create_mr_response.body)["iid"] diff_refs = JSON.parse(create_mr_response.body)["diff_refs"] @@ -161,8 +164,8 @@ module QA if should_resolve discussion_id = JSON.parse(create_diff_note_response.body)["id"] - put Runtime::API::Request.new(@api_client, """/projects/#{@group_name}%2F#{@project_name}/merge_requests/#{iid}/discussions/#{discussion_id}""").url, - "resolved=true" + + update_a_discussion_on_issue_api_req("#{@group_name}%2F#{@project_name}", iid, discussion_id, "true") end should_resolve ^= true @@ -171,23 +174,93 @@ module QA # Add discussions to main tab 100.times do - post Runtime::API::Request.new(@api_client, "/projects/#{@group_name}%2F#{@project_name}/merge_requests/#{iid}/discussions").url, - "body=\"Let us discuss\"" + create_a_discussion_on_mr_api_req("#{@group_name}%2F#{@project_name}", iid, "Let us discuss") end @urls[:large_mr] = JSON.parse(create_mr_response.body)["web_url"] - STDOUT.puts "Created MR with 500 Discussions and 20 Very Large Files" + STDOUT.puts "Created an MR with many discussions and many very large Files: #{@urls[:large_mr]}" end def create_diff_note(iid, file_count, line_count, head_sha, start_sha, base_sha, line_type) post Runtime::API::Request.new(@api_client, "/projects/#{@group_name}%2F#{@project_name}/merge_requests/#{iid}/discussions").url, - """body=\"Let us discuss\"& + "" "body=\"Let us discuss\"& position[position_type]=text& position[new_path]=hello#{file_count}.txt& position[old_path]=hello#{file_count}.txt& position[#{line_type}]=#{line_count * 100}& position[head_sha]=#{head_sha}& position[start_sha]=#{start_sha}& - position[base_sha]=#{base_sha}""" + position[base_sha]=#{base_sha}" "" + end + + def create_mr_with_many_commits + project_path = "#{@group_name}%2F#{@project_name}" + branch_name = "branch_with_many_commits-#{SecureRandom.hex(8)}" + file_name = "file_for_many_commits.txt" + create_a_branch_api_req(branch_name, project_path) + create_a_new_file_api_req(file_name, branch_name, project_path, "Initial commit for new file", "Initial file content") + create_mr_response = create_a_merge_request_api_req(project_path, branch_name, "master", "MR with many commits-#{SecureRandom.hex(8)}") + @urls[:mr_with_many_commits] = JSON.parse(create_mr_response.body)["web_url"] + 100.times do |i| + update_file_api_req(file_name, branch_name, project_path, Faker::Lorem.sentences(5).join(" "), Faker::Lorem.sentences(500).join("\n")) + end + STDOUT.puts "Created an MR with many commits: #{@urls[:mr_with_many_commits]}" + end + + private + + # API Requests + + def create_a_discussion_on_issue_api_req(project_path_or_id, issue_id, body) + post Runtime::API::Request.new(@api_client, "/projects/#{project_path_or_id}/issues/#{issue_id}/discussions").url, "body=\"#{body}\"" + end + + def update_a_discussion_on_issue_api_req(project_path_or_id, mr_iid, discussion_id, resolved_status) + put Runtime::API::Request.new(@api_client, "/projects/#{project_path_or_id}/merge_requests/#{mr_iid}/discussions/#{discussion_id}").url, "resolved=#{resolved_status}" + end + + def create_a_discussion_on_mr_api_req(project_path_or_id, mr_iid, body) + post Runtime::API::Request.new(@api_client, "/projects/#{project_path_or_id}/merge_requests/#{mr_iid}/discussions").url, + "body=\"#{body}\"" + end + + def create_a_label_api_req(project_path_or_id, name, color) + post Runtime::API::Request.new(@api_client, "/projects/#{project_path_or_id}/labels").url, "name=#{name}&color=#{color}" + end + + def create_a_todo_api_req(project_path_or_id, issue_id) + post Runtime::API::Request.new(@api_client, "/projects/#{project_path_or_id}/issues/#{issue_id}/todo").url, nil + end + + def create_an_issue_api_req(project_path_or_id, title, description) + post Runtime::API::Request.new(@api_client, "/projects/#{project_path_or_id}/issues").url, "title=#{title}&description=#{description}" + end + + def update_an_issue_api_req(project_path_or_id, issue_id, description, labels_list) + put Runtime::API::Request.new(@api_client, "/projects/#{project_path_or_id}/issues/#{issue_id}").url, "description=#{description}&labels=#{labels_list}" + end + + def create_a_project_api_req(project_name, group_id, visibility) + post Runtime::API::Request.new(@api_client, "/projects").url, "name=#{project_name}&namespace_id=#{group_id}&visibility=#{visibility}" + end + + def create_a_group_api_req(group_name, visibility) + post Runtime::API::Request.new(@api_client, "/groups").url, "name=#{group_name}&path=#{group_name}&visibility=#{visibility}" + end + + def create_a_branch_api_req(branch_name, project_path_or_id) + post Runtime::API::Request.new(@api_client, "/projects/#{project_path_or_id}/repository/branches").url, "branch=#{branch_name}&ref=master" + end + + def create_a_new_file_api_req(file_path, branch_name, project_path_or_id, commit_message, content) + post Runtime::API::Request.new(@api_client, "/projects/#{project_path_or_id}/repository/files/#{file_path}").url, "branch=#{branch_name}&commit_message=\"#{commit_message}\"&content=\"#{content}\"" + end + + def create_a_merge_request_api_req(project_path_or_id, source_branch, target_branch, mr_title) + post Runtime::API::Request.new(@api_client, "/projects/#{project_path_or_id}/merge_requests").url, "source_branch=#{source_branch}&target_branch=#{target_branch}&title=#{mr_title}" + end + + def update_file_api_req(file_path, branch_name, project_path_or_id, commit_message, content) + put Runtime::API::Request.new(@api_client, "/projects/#{project_path_or_id}/repository/files/#{file_path}").url, "branch=#{branch_name}&commit_message=\"#{commit_message}\"&content=\"#{content}\"" end end end