2016-04-14 15:10:57 +00:00
|
|
|
module Gitlab
|
2016-03-09 15:21:02 +00:00
|
|
|
module ImportExport
|
|
|
|
class ProjectTreeRestorer
|
2017-08-17 10:40:19 +00:00
|
|
|
# Relations which cannot have both group_id and project_id at the same time
|
2017-10-03 13:52:52 +00:00
|
|
|
RESTRICT_PROJECT_AND_GROUP = %i(milestone milestones).freeze
|
2017-08-17 10:40:19 +00:00
|
|
|
|
2016-06-14 10:47:07 +00:00
|
|
|
def initialize(user:, shared:, project:)
|
2016-05-11 15:22:45 +00:00
|
|
|
@path = File.join(shared.export_path, 'project.json')
|
2016-03-09 17:42:04 +00:00
|
|
|
@user = user
|
2016-05-11 15:22:45 +00:00
|
|
|
@shared = shared
|
2016-06-14 10:47:07 +00:00
|
|
|
@project = project
|
2017-09-01 15:20:09 +00:00
|
|
|
@project_id = project.id
|
2017-09-06 06:56:38 +00:00
|
|
|
@saved = true
|
2016-03-09 15:21:02 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
def restore
|
2016-10-27 15:10:19 +00:00
|
|
|
begin
|
|
|
|
json = IO.read(@path)
|
|
|
|
@tree_hash = ActiveSupport::JSON.decode(json)
|
|
|
|
rescue => e
|
|
|
|
Rails.logger.error("Import/Export error: #{e.message}")
|
|
|
|
raise Gitlab::ImportExport::Error.new('Incorrect JSON format')
|
|
|
|
end
|
|
|
|
|
2016-04-11 16:30:54 +00:00
|
|
|
@project_members = @tree_hash.delete('project_members')
|
2016-07-14 14:03:00 +00:00
|
|
|
|
2017-09-03 18:51:50 +00:00
|
|
|
ActiveRecord::Base.uncached do
|
|
|
|
ActiveRecord::Base.no_touching do
|
2017-09-03 17:16:18 +00:00
|
|
|
create_relations
|
|
|
|
end
|
2016-07-14 14:03:00 +00:00
|
|
|
end
|
2016-05-06 13:18:25 +00:00
|
|
|
rescue => e
|
2016-05-13 10:33:13 +00:00
|
|
|
@shared.error(e)
|
2016-05-05 16:12:24 +00:00
|
|
|
false
|
2016-03-10 17:43:57 +00:00
|
|
|
end
|
|
|
|
|
2016-06-14 18:32:19 +00:00
|
|
|
def restored_project
|
2016-06-14 10:47:07 +00:00
|
|
|
@restored_project ||= restore_project
|
2016-05-03 10:41:23 +00:00
|
|
|
end
|
|
|
|
|
2016-03-10 17:43:57 +00:00
|
|
|
private
|
|
|
|
|
2016-05-13 10:33:13 +00:00
|
|
|
def members_mapper
|
|
|
|
@members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members,
|
|
|
|
user: @user,
|
2016-06-14 18:32:19 +00:00
|
|
|
project: restored_project)
|
2016-03-10 17:43:57 +00:00
|
|
|
end
|
|
|
|
|
2016-06-02 12:07:09 +00:00
|
|
|
# Loops through the tree of models defined in import_export.yml and
|
|
|
|
# finds them in the imported JSON so they can be instantiated and saved
|
|
|
|
# in the DB. The structure and relationships between models are guessed from
|
|
|
|
# the configuration yaml file too.
|
|
|
|
# Finally, it updates each attribute in the newly imported project.
|
2016-06-01 16:03:51 +00:00
|
|
|
def create_relations
|
|
|
|
default_relation_list.each do |relation|
|
2017-09-01 15:20:09 +00:00
|
|
|
if relation.is_a?(Hash)
|
|
|
|
create_sub_relations(relation, @tree_hash)
|
2017-09-06 08:16:11 +00:00
|
|
|
elsif @tree_hash[relation.to_s].present?
|
2017-09-06 09:11:02 +00:00
|
|
|
save_relation_hash(@tree_hash[relation.to_s], relation)
|
2017-09-01 15:20:09 +00:00
|
|
|
end
|
|
|
|
end
|
2017-09-05 15:24:57 +00:00
|
|
|
|
2017-09-06 06:56:38 +00:00
|
|
|
@saved
|
2017-09-01 15:20:09 +00:00
|
|
|
end
|
2017-04-06 14:47:13 +00:00
|
|
|
|
2017-09-01 15:20:09 +00:00
|
|
|
def save_relation_hash(relation_hash_batch, relation_key)
|
|
|
|
relation_hash = create_relation(relation_key, relation_hash_batch)
|
2017-09-02 14:17:41 +00:00
|
|
|
|
2017-09-06 06:56:38 +00:00
|
|
|
@saved = false unless restored_project.append_or_update_attribute(relation_key, relation_hash)
|
2017-09-05 15:24:57 +00:00
|
|
|
|
2017-09-06 06:56:38 +00:00
|
|
|
# Restore the project again, extra query that skips holding the AR objects in memory
|
2017-09-06 08:09:24 +00:00
|
|
|
@restored_project = Project.find(@project_id)
|
2016-03-14 17:32:56 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
def default_relation_list
|
2016-06-14 14:37:41 +00:00
|
|
|
Gitlab::ImportExport::Reader.new(shared: @shared).tree.reject do |model|
|
2016-05-18 13:15:14 +00:00
|
|
|
model.is_a?(Hash) && model[:project_members]
|
2016-05-11 15:22:45 +00:00
|
|
|
end
|
2016-03-09 15:21:02 +00:00
|
|
|
end
|
|
|
|
|
2016-06-14 10:47:07 +00:00
|
|
|
def restore_project
|
2016-06-14 18:32:19 +00:00
|
|
|
return @project unless @tree_hash
|
|
|
|
|
2017-05-03 10:12:32 +00:00
|
|
|
@project.update_columns(project_params)
|
2016-06-14 10:47:07 +00:00
|
|
|
@project
|
2016-03-10 15:21:17 +00:00
|
|
|
end
|
2017-09-01 15:20:09 +00:00
|
|
|
|
2016-09-26 09:32:26 +00:00
|
|
|
def project_params
|
|
|
|
@tree_hash.reject do |key, value|
|
|
|
|
# return params that are not 1 to many or 1 to 1 relations
|
2017-05-03 10:12:32 +00:00
|
|
|
value.respond_to?(:each) && !Project.column_names.include?(key)
|
2016-09-26 09:32:26 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2016-06-13 14:55:51 +00:00
|
|
|
# Given a relation hash containing one or more models and its relationships,
|
|
|
|
# loops through each model and each object from a model type and
|
|
|
|
# and assigns its correspondent attributes hash from +tree_hash+
|
|
|
|
# Example:
|
|
|
|
# +relation_key+ issues, loops through the list of *issues* and for each individual
|
|
|
|
# issue, finds any subrelations such as notes, creates them and assign them back to the hash
|
2016-07-08 15:21:28 +00:00
|
|
|
#
|
|
|
|
# Recursively calls this method if the sub-relation is a hash containing more sub-relations
|
2017-09-06 09:11:02 +00:00
|
|
|
def create_sub_relations(relation, tree_hash, save: true)
|
2016-05-13 10:33:13 +00:00
|
|
|
relation_key = relation.keys.first.to_s
|
2016-07-08 15:21:28 +00:00
|
|
|
return if tree_hash[relation_key].blank?
|
|
|
|
|
2017-09-03 17:16:18 +00:00
|
|
|
tree_array = [tree_hash[relation_key]].flatten
|
2016-07-08 15:21:28 +00:00
|
|
|
|
2017-09-05 15:24:57 +00:00
|
|
|
# Avoid keeping a possible heavy object in memory once we are done with it
|
2017-09-03 17:16:18 +00:00
|
|
|
while relation_item = tree_array.shift
|
2017-09-05 15:24:57 +00:00
|
|
|
# The transaction at this level is less speedy than one single transaction
|
|
|
|
# But we can't have it in the upper level or GC won't get rid of the AR objects
|
|
|
|
# after we save the batch.
|
2017-09-03 18:51:50 +00:00
|
|
|
Project.transaction do
|
2017-09-03 18:01:14 +00:00
|
|
|
process_sub_relation(relation, relation_item)
|
2017-09-03 17:16:18 +00:00
|
|
|
|
2017-09-05 15:24:57 +00:00
|
|
|
# For every subrelation that hangs from Project, save the associated records alltogether
|
|
|
|
# This effectively batches all records per subrelation item, only keeping those in memory
|
|
|
|
# We have to keep in mind that more batch granularity << Memory, but >> Slowness
|
2017-09-03 17:16:18 +00:00
|
|
|
if save
|
|
|
|
save_relation_hash([relation_item], relation_key)
|
2017-09-03 18:01:14 +00:00
|
|
|
tree_hash[relation_key].delete(relation_item)
|
2017-09-03 17:16:18 +00:00
|
|
|
end
|
2017-09-03 18:51:50 +00:00
|
|
|
end
|
2016-04-11 16:30:54 +00:00
|
|
|
end
|
2017-09-03 17:16:18 +00:00
|
|
|
|
|
|
|
tree_hash.delete(relation_key) if save
|
2016-04-11 16:30:54 +00:00
|
|
|
end
|
|
|
|
|
2017-09-03 18:01:14 +00:00
|
|
|
def process_sub_relation(relation, relation_item)
|
|
|
|
relation.values.flatten.each do |sub_relation|
|
|
|
|
# We just use author to get the user ID, do not attempt to create an instance.
|
|
|
|
next if sub_relation == :author
|
|
|
|
|
2017-09-06 09:11:02 +00:00
|
|
|
create_sub_relations(sub_relation, relation_item, save: false) if sub_relation.is_a?(Hash)
|
2017-09-03 18:01:14 +00:00
|
|
|
|
|
|
|
relation_hash, sub_relation = assign_relation_hash(relation_item, sub_relation)
|
|
|
|
relation_item[sub_relation.to_s] = create_relation(sub_relation, relation_hash) unless relation_hash.blank?
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2016-06-13 14:55:51 +00:00
|
|
|
def assign_relation_hash(relation_item, sub_relation)
|
|
|
|
if sub_relation.is_a?(Hash)
|
|
|
|
relation_hash = relation_item[sub_relation.keys.first.to_s]
|
|
|
|
sub_relation = sub_relation.keys.first
|
|
|
|
else
|
|
|
|
relation_hash = relation_item[sub_relation.to_s]
|
|
|
|
end
|
2016-06-14 08:20:47 +00:00
|
|
|
[relation_hash, sub_relation]
|
2016-06-13 14:55:51 +00:00
|
|
|
end
|
|
|
|
|
2016-03-10 17:43:57 +00:00
|
|
|
def create_relation(relation, relation_hash_list)
|
2016-06-02 08:59:54 +00:00
|
|
|
relation_array = [relation_hash_list].flatten.map do |relation_hash|
|
2017-09-05 19:06:27 +00:00
|
|
|
Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym,
|
|
|
|
relation_hash: parsed_relation_hash(relation_hash, relation.to_sym),
|
2016-06-01 16:03:51 +00:00
|
|
|
members_mapper: members_mapper,
|
2016-08-12 10:04:33 +00:00
|
|
|
user: @user,
|
2017-09-05 19:06:27 +00:00
|
|
|
project: @restored_project)
|
2016-12-16 15:13:46 +00:00
|
|
|
end.compact
|
2016-06-02 08:59:54 +00:00
|
|
|
|
|
|
|
relation_hash_list.is_a?(Array) ? relation_array : relation_array.first
|
2016-03-09 15:21:02 +00:00
|
|
|
end
|
2016-09-29 11:15:18 +00:00
|
|
|
|
2017-08-17 10:40:19 +00:00
|
|
|
def parsed_relation_hash(relation_hash, relation_type)
|
|
|
|
if RESTRICT_PROJECT_AND_GROUP.include?(relation_type)
|
|
|
|
params = {}
|
|
|
|
params['group_id'] = restored_project.group.try(:id) if relation_hash['group_id']
|
|
|
|
params['project_id'] = restored_project.id if relation_hash['project_id']
|
|
|
|
else
|
|
|
|
params = { 'group_id' => restored_project.group.try(:id), 'project_id' => restored_project.id }
|
|
|
|
end
|
|
|
|
|
|
|
|
relation_hash.merge(params)
|
2016-09-29 11:15:18 +00:00
|
|
|
end
|
2016-03-09 15:21:02 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|