Merge branch 'gh-rate-limit'
This commit is contained in:
commit
affd58fb79
7 changed files with 97 additions and 36 deletions
|
@ -49,6 +49,7 @@ v 8.8.4 (unreleased)
|
|||
- Reduce number of SQL queries when rendering user references
|
||||
- Upgrade to jQuery 2
|
||||
- Remove prev/next buttons on issues and merge requests
|
||||
- Import GitHub repositories respecting the API rate limit
|
||||
|
||||
v 8.8.3
|
||||
- Fix 404 page when viewing TODOs that contain milestones or labels in different projects. !4312
|
||||
|
|
|
@ -9,6 +9,10 @@ module Gitlab
|
|||
@formatter = Gitlab::ImportFormatter.new
|
||||
end
|
||||
|
||||
def create!
|
||||
self.klass.create!(self.attributes)
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def gl_user_id(github_id)
|
||||
|
|
|
@ -3,6 +3,9 @@ module Gitlab
|
|||
class Importer
|
||||
include Gitlab::ShellAdapter
|
||||
|
||||
GITHUB_SAFE_REMAINING_REQUESTS = 100
|
||||
GITHUB_SAFE_SLEEP_TIME = 500
|
||||
|
||||
attr_reader :client, :project, :repo, :repo_url
|
||||
|
||||
def initialize(project)
|
||||
|
@ -25,14 +28,53 @@ module Gitlab
|
|||
|
||||
private
|
||||
|
||||
def turn_auto_pagination_off!
|
||||
client.auto_paginate = false
|
||||
end
|
||||
|
||||
def turn_auto_pagination_on!
|
||||
client.auto_paginate = true
|
||||
end
|
||||
|
||||
def rate_limit
|
||||
client.rate_limit!
|
||||
end
|
||||
|
||||
def rate_limit_exceed?
|
||||
rate_limit.remaining <= GITHUB_SAFE_REMAINING_REQUESTS
|
||||
end
|
||||
|
||||
def rate_limit_sleep_time
|
||||
rate_limit.resets_in + GITHUB_SAFE_SLEEP_TIME
|
||||
end
|
||||
|
||||
def paginate
|
||||
turn_auto_pagination_off!
|
||||
|
||||
sleep rate_limit_sleep_time if rate_limit_exceed?
|
||||
|
||||
data = yield
|
||||
|
||||
last_response = client.last_response
|
||||
|
||||
while last_response.rels[:next]
|
||||
sleep rate_limit_sleep_time if rate_limit_exceed?
|
||||
last_response = last_response.rels[:next].get
|
||||
data.concat(last_response.data) if last_response.data.is_a?(Array)
|
||||
end
|
||||
|
||||
turn_auto_pagination_on!
|
||||
|
||||
data
|
||||
end
|
||||
|
||||
def credentials
|
||||
@credentials ||= project.import_data.credentials if project.import_data
|
||||
end
|
||||
|
||||
def import_labels
|
||||
client.labels(repo).each do |raw_data|
|
||||
Label.create!(LabelFormatter.new(project, raw_data).attributes)
|
||||
end
|
||||
labels = paginate { client.labels(repo, per_page: 100) }
|
||||
labels.each { |raw| LabelFormatter.new(project, raw).create! }
|
||||
|
||||
true
|
||||
rescue ActiveRecord::RecordInvalid => e
|
||||
|
@ -40,9 +82,8 @@ module Gitlab
|
|||
end
|
||||
|
||||
def import_milestones
|
||||
client.list_milestones(repo, state: :all).each do |raw_data|
|
||||
Milestone.create!(MilestoneFormatter.new(project, raw_data).attributes)
|
||||
end
|
||||
milestones = paginate { client.milestones(repo, state: :all, per_page: 100) }
|
||||
milestones.each { |raw| MilestoneFormatter.new(project, raw).create! }
|
||||
|
||||
true
|
||||
rescue ActiveRecord::RecordInvalid => e
|
||||
|
@ -50,16 +91,15 @@ module Gitlab
|
|||
end
|
||||
|
||||
def import_issues
|
||||
client.list_issues(repo, state: :all, sort: :created, direction: :asc).each do |raw_data|
|
||||
gh_issue = IssueFormatter.new(project, raw_data)
|
||||
data = paginate { client.issues(repo, state: :all, sort: :created, direction: :asc, per_page: 100) }
|
||||
|
||||
data.each do |raw|
|
||||
gh_issue = IssueFormatter.new(project, raw)
|
||||
|
||||
if gh_issue.valid?
|
||||
issue = Issue.create!(gh_issue.attributes)
|
||||
apply_labels(gh_issue.number, issue)
|
||||
|
||||
if gh_issue.has_comments?
|
||||
import_comments(gh_issue.number, issue)
|
||||
end
|
||||
issue = gh_issue.create!
|
||||
apply_labels(issue)
|
||||
import_comments(issue) if gh_issue.has_comments?
|
||||
end
|
||||
end
|
||||
|
||||
|
@ -69,9 +109,8 @@ module Gitlab
|
|||
end
|
||||
|
||||
def import_pull_requests
|
||||
pull_requests = client.pull_requests(repo, state: :all, sort: :created, direction: :asc)
|
||||
.map { |raw| PullRequestFormatter.new(project, raw) }
|
||||
.select(&:valid?)
|
||||
pull_requests = paginate { client.pull_requests(repo, state: :all, sort: :created, direction: :asc, per_page: 100) }
|
||||
pull_requests = pull_requests.map { |raw| PullRequestFormatter.new(project, raw) }.select(&:valid?)
|
||||
|
||||
source_branches_removed = pull_requests.reject(&:source_branch_exists?).map { |pr| [pr.source_branch_name, pr.source_branch_sha] }
|
||||
target_branches_removed = pull_requests.reject(&:target_branch_exists?).map { |pr| [pr.target_branch_name, pr.target_branch_sha] }
|
||||
|
@ -80,13 +119,10 @@ module Gitlab
|
|||
create_refs(branches_removed)
|
||||
|
||||
pull_requests.each do |pull_request|
|
||||
merge_request = MergeRequest.new(pull_request.attributes)
|
||||
|
||||
if merge_request.save
|
||||
apply_labels(pull_request.number, merge_request)
|
||||
import_comments(pull_request.number, merge_request)
|
||||
import_comments_on_diff(pull_request.number, merge_request)
|
||||
end
|
||||
merge_request = pull_request.create!
|
||||
apply_labels(merge_request)
|
||||
import_comments(merge_request)
|
||||
import_comments_on_diff(merge_request)
|
||||
end
|
||||
|
||||
true
|
||||
|
@ -98,6 +134,7 @@ module Gitlab
|
|||
|
||||
def create_refs(branches)
|
||||
branches.each do |name, sha|
|
||||
sleep rate_limit_sleep_time if rate_limit_exceed?
|
||||
client.create_ref(repo, "refs/heads/#{name}", sha)
|
||||
end
|
||||
|
||||
|
@ -106,13 +143,16 @@ module Gitlab
|
|||
|
||||
def delete_refs(branches)
|
||||
branches.each do |name, _|
|
||||
sleep rate_limit_sleep_time if rate_limit_exceed?
|
||||
client.delete_ref(repo, "heads/#{name}")
|
||||
project.repository.rm_branch(project.creator, name)
|
||||
end
|
||||
end
|
||||
|
||||
def apply_labels(number, issuable)
|
||||
issue = client.issue(repo, number)
|
||||
def apply_labels(issuable)
|
||||
sleep rate_limit_sleep_time if rate_limit_exceed?
|
||||
|
||||
issue = client.issue(repo, issuable.iid)
|
||||
|
||||
if issue.labels.count > 0
|
||||
label_ids = issue.labels.map do |raw|
|
||||
|
@ -123,20 +163,20 @@ module Gitlab
|
|||
end
|
||||
end
|
||||
|
||||
def import_comments(issue_number, noteable)
|
||||
comments = client.issue_comments(repo, issue_number)
|
||||
create_comments(comments, noteable)
|
||||
def import_comments(issuable)
|
||||
comments = paginate { client.issue_comments(repo, issuable.iid, per_page: 100) }
|
||||
create_comments(issuable, comments)
|
||||
end
|
||||
|
||||
def import_comments_on_diff(pull_request_number, merge_request)
|
||||
comments = client.pull_request_comments(repo, pull_request_number)
|
||||
create_comments(comments, merge_request)
|
||||
def import_comments_on_diff(merge_request)
|
||||
comments = paginate { client.pull_request_comments(repo, merge_request.iid, per_page: 100) }
|
||||
create_comments(merge_request, comments)
|
||||
end
|
||||
|
||||
def create_comments(comments, noteable)
|
||||
comments.each do |raw_data|
|
||||
comment = CommentFormatter.new(project, raw_data)
|
||||
noteable.notes.create!(comment.attributes)
|
||||
def create_comments(issuable, comments)
|
||||
comments.each do |raw|
|
||||
comment = CommentFormatter.new(project, raw)
|
||||
issuable.notes.create!(comment.attributes)
|
||||
end
|
||||
end
|
||||
|
||||
|
|
|
@ -20,6 +20,10 @@ module Gitlab
|
|||
raw_data.comments > 0
|
||||
end
|
||||
|
||||
def klass
|
||||
Issue
|
||||
end
|
||||
|
||||
def number
|
||||
raw_data.number
|
||||
end
|
||||
|
|
|
@ -9,6 +9,10 @@ module Gitlab
|
|||
}
|
||||
end
|
||||
|
||||
def klass
|
||||
Label
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def color
|
||||
|
|
|
@ -14,6 +14,10 @@ module Gitlab
|
|||
}
|
||||
end
|
||||
|
||||
def klass
|
||||
Milestone
|
||||
end
|
||||
|
||||
private
|
||||
|
||||
def number
|
||||
|
|
|
@ -24,6 +24,10 @@ module Gitlab
|
|||
}
|
||||
end
|
||||
|
||||
def klass
|
||||
MergeRequest
|
||||
end
|
||||
|
||||
def number
|
||||
raw_data.number
|
||||
end
|
||||
|
|
Loading…
Reference in a new issue