2018-11-17 00:37:17 +00:00
|
|
|
# frozen_string_literal: true
|
|
|
|
|
2016-04-14 15:10:57 +00:00
|
|
|
module Gitlab
|
2016-03-09 15:21:02 +00:00
|
|
|
module ImportExport
|
|
|
|
class ProjectTreeRestorer
|
2018-06-25 07:42:07 +00:00
|
|
|
# Relations which cannot be saved at project level (and have a group assigned)
|
2018-06-19 15:08:02 +00:00
|
|
|
GROUP_MODELS = [GroupLabel, Milestone].freeze
|
2017-08-17 10:40:19 +00:00
|
|
|
|
2016-06-14 10:47:07 +00:00
|
|
|
def initialize(user:, shared:, project:)
|
2016-05-11 15:22:45 +00:00
|
|
|
@path = File.join(shared.export_path, 'project.json')
|
2016-03-09 17:42:04 +00:00
|
|
|
@user = user
|
2016-05-11 15:22:45 +00:00
|
|
|
@shared = shared
|
2016-06-14 10:47:07 +00:00
|
|
|
@project = project
|
2017-09-01 15:20:09 +00:00
|
|
|
@project_id = project.id
|
2017-09-06 06:56:38 +00:00
|
|
|
@saved = true
|
2016-03-09 15:21:02 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
def restore
|
2016-10-27 15:10:19 +00:00
|
|
|
begin
|
|
|
|
json = IO.read(@path)
|
|
|
|
@tree_hash = ActiveSupport::JSON.decode(json)
|
|
|
|
rescue => e
|
|
|
|
Rails.logger.error("Import/Export error: #{e.message}")
|
|
|
|
raise Gitlab::ImportExport::Error.new('Incorrect JSON format')
|
|
|
|
end
|
|
|
|
|
2016-04-11 16:30:54 +00:00
|
|
|
@project_members = @tree_hash.delete('project_members')
|
2016-07-14 14:03:00 +00:00
|
|
|
|
2018-12-05 14:39:15 +00:00
|
|
|
RelationRenameService.rename(@tree_hash)
|
|
|
|
|
2017-09-03 18:51:50 +00:00
|
|
|
ActiveRecord::Base.uncached do
|
|
|
|
ActiveRecord::Base.no_touching do
|
2017-09-03 17:16:18 +00:00
|
|
|
create_relations
|
|
|
|
end
|
2016-07-14 14:03:00 +00:00
|
|
|
end
|
2016-05-06 13:18:25 +00:00
|
|
|
rescue => e
|
2016-05-13 10:33:13 +00:00
|
|
|
@shared.error(e)
|
2016-05-05 16:12:24 +00:00
|
|
|
false
|
2016-03-10 17:43:57 +00:00
|
|
|
end
|
|
|
|
|
2016-06-14 18:32:19 +00:00
|
|
|
def restored_project
|
2018-03-14 14:17:35 +00:00
|
|
|
return @project unless @tree_hash
|
|
|
|
|
2016-06-14 10:47:07 +00:00
|
|
|
@restored_project ||= restore_project
|
2016-05-03 10:41:23 +00:00
|
|
|
end
|
|
|
|
|
2016-03-10 17:43:57 +00:00
|
|
|
private
|
|
|
|
|
2016-05-13 10:33:13 +00:00
|
|
|
def members_mapper
|
|
|
|
@members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members,
|
|
|
|
user: @user,
|
2016-06-14 18:32:19 +00:00
|
|
|
project: restored_project)
|
2016-03-10 17:43:57 +00:00
|
|
|
end
|
|
|
|
|
2016-06-02 12:07:09 +00:00
|
|
|
# Loops through the tree of models defined in import_export.yml and
|
|
|
|
# finds them in the imported JSON so they can be instantiated and saved
|
|
|
|
# in the DB. The structure and relationships between models are guessed from
|
|
|
|
# the configuration yaml file too.
|
|
|
|
# Finally, it updates each attribute in the newly imported project.
|
2016-06-01 16:03:51 +00:00
|
|
|
def create_relations
|
|
|
|
default_relation_list.each do |relation|
|
2017-09-01 15:20:09 +00:00
|
|
|
if relation.is_a?(Hash)
|
|
|
|
create_sub_relations(relation, @tree_hash)
|
2017-09-06 08:16:11 +00:00
|
|
|
elsif @tree_hash[relation.to_s].present?
|
2017-09-06 09:11:02 +00:00
|
|
|
save_relation_hash(@tree_hash[relation.to_s], relation)
|
2017-09-01 15:20:09 +00:00
|
|
|
end
|
|
|
|
end
|
2017-09-05 15:24:57 +00:00
|
|
|
|
Use latest_merge_request_diff association
Compared to the merge_request_diff association:
1. It's simpler to query. The query uses a foreign key to the
merge_request_diffs table, so no ordering is necessary.
2. It's faster for preloading. The merge_request_diff association has to load
every diff for the MRs in the set, then discard all but the most recent for
each. This association means that Rails can just query for N diffs from N
MRs.
3. It's more complicated to update. This is a bidirectional foreign key, so we
need to update two tables when adding a diff record. This also means we need
to handle this as a special case when importing a GitLab project.
There is some juggling with this association in the merge request model:
* `MergeRequest#latest_merge_request_diff` is _always_ the latest diff.
* `MergeRequest#merge_request_diff` reuses
`MergeRequest#latest_merge_request_diff` unless:
* Arguments are passed. These are typically to force-reload the association.
* It doesn't exist. That means we might be trying to implicitly create a
diff. This only seems to happen in specs.
* The association is already loaded. This is important for the reasons
explained in the comment, which I'll reiterate here: if we a) load a
non-latest diff, then b) get its `merge_request`, then c) get that MR's
`merge_request_diff`, we should get the diff we loaded in c), even though
that's not the latest diff.
Basically, `MergeRequest#merge_request_diff` is the latest diff in most cases,
but not quite all.
2017-11-15 17:22:18 +00:00
|
|
|
@project.merge_requests.set_latest_merge_request_diff_ids!
|
|
|
|
|
2017-09-06 06:56:38 +00:00
|
|
|
@saved
|
2017-09-01 15:20:09 +00:00
|
|
|
end
|
2017-04-06 14:47:13 +00:00
|
|
|
|
2017-09-01 15:20:09 +00:00
|
|
|
def save_relation_hash(relation_hash_batch, relation_key)
|
|
|
|
relation_hash = create_relation(relation_key, relation_hash_batch)
|
2017-09-02 14:17:41 +00:00
|
|
|
|
2018-06-19 15:08:02 +00:00
|
|
|
remove_group_models(relation_hash) if relation_hash.is_a?(Array)
|
|
|
|
|
2017-09-06 06:56:38 +00:00
|
|
|
@saved = false unless restored_project.append_or_update_attribute(relation_key, relation_hash)
|
2017-09-05 15:24:57 +00:00
|
|
|
|
2017-09-06 06:56:38 +00:00
|
|
|
# Restore the project again, extra query that skips holding the AR objects in memory
|
2017-09-06 08:09:24 +00:00
|
|
|
@restored_project = Project.find(@project_id)
|
2016-03-14 17:32:56 +00:00
|
|
|
end
|
|
|
|
|
2018-06-22 09:27:52 +00:00
|
|
|
# Remove project models that became group models as we found them at group level.
|
|
|
|
# This no longer required saving them at the root project level.
|
|
|
|
# For example, in the case of an existing group label that matched the title.
|
2018-06-19 15:08:02 +00:00
|
|
|
def remove_group_models(relation_hash)
|
|
|
|
relation_hash.reject! do |value|
|
2018-06-25 07:42:07 +00:00
|
|
|
GROUP_MODELS.include?(value.class) && value.group_id
|
2018-06-19 15:08:02 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2016-03-14 17:32:56 +00:00
|
|
|
def default_relation_list
|
2018-03-29 13:08:31 +00:00
|
|
|
reader.tree.reject do |model|
|
2016-05-18 13:15:14 +00:00
|
|
|
model.is_a?(Hash) && model[:project_members]
|
2016-05-11 15:22:45 +00:00
|
|
|
end
|
2016-03-09 15:21:02 +00:00
|
|
|
end
|
|
|
|
|
2016-06-14 10:47:07 +00:00
|
|
|
def restore_project
|
2018-08-16 10:02:34 +00:00
|
|
|
Gitlab::Timeless.timeless(@project) do
|
|
|
|
@project.update(project_params)
|
|
|
|
end
|
|
|
|
|
2016-06-14 10:47:07 +00:00
|
|
|
@project
|
2016-03-10 15:21:17 +00:00
|
|
|
end
|
2017-09-01 15:20:09 +00:00
|
|
|
|
2016-09-26 09:32:26 +00:00
|
|
|
def project_params
|
2018-05-03 09:02:26 +00:00
|
|
|
@project_params ||= begin
|
2018-12-19 11:49:59 +00:00
|
|
|
attrs = json_params.merge(override_params).merge(visibility_level)
|
2018-05-03 09:02:26 +00:00
|
|
|
|
|
|
|
# Cleaning all imported and overridden params
|
|
|
|
Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: attrs,
|
|
|
|
relation_class: Project,
|
|
|
|
excluded_keys: excluded_keys_for_relation(:project))
|
|
|
|
end
|
2018-03-29 13:08:31 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
def override_params
|
2018-05-03 09:02:26 +00:00
|
|
|
@override_params ||= @project.import_data&.data&.fetch('override_params', nil) || {}
|
2018-03-29 13:08:31 +00:00
|
|
|
end
|
|
|
|
|
|
|
|
def json_params
|
|
|
|
@json_params ||= @tree_hash.reject do |key, value|
|
2016-09-26 09:32:26 +00:00
|
|
|
# return params that are not 1 to many or 1 to 1 relations
|
2017-05-03 10:12:32 +00:00
|
|
|
value.respond_to?(:each) && !Project.column_names.include?(key)
|
2016-09-26 09:32:26 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2018-12-19 11:49:59 +00:00
|
|
|
def visibility_level
|
|
|
|
level = override_params['visibility_level'] || json_params['visibility_level'] || @project.visibility_level
|
|
|
|
level = @project.group.visibility_level if @project.group && level > @project.group.visibility_level
|
|
|
|
|
|
|
|
{ 'visibility_level' => level }
|
|
|
|
end
|
|
|
|
|
2016-06-13 14:55:51 +00:00
|
|
|
# Given a relation hash containing one or more models and its relationships,
|
|
|
|
# loops through each model and each object from a model type and
|
|
|
|
# and assigns its correspondent attributes hash from +tree_hash+
|
|
|
|
# Example:
|
|
|
|
# +relation_key+ issues, loops through the list of *issues* and for each individual
|
|
|
|
# issue, finds any subrelations such as notes, creates them and assign them back to the hash
|
2016-07-08 15:21:28 +00:00
|
|
|
#
|
|
|
|
# Recursively calls this method if the sub-relation is a hash containing more sub-relations
|
2017-09-06 09:11:02 +00:00
|
|
|
def create_sub_relations(relation, tree_hash, save: true)
|
2016-05-13 10:33:13 +00:00
|
|
|
relation_key = relation.keys.first.to_s
|
2016-07-08 15:21:28 +00:00
|
|
|
return if tree_hash[relation_key].blank?
|
|
|
|
|
2017-09-03 17:16:18 +00:00
|
|
|
tree_array = [tree_hash[relation_key]].flatten
|
2018-09-24 06:01:03 +00:00
|
|
|
null_iid_pipelines = []
|
2016-07-08 15:21:28 +00:00
|
|
|
|
2017-09-05 15:24:57 +00:00
|
|
|
# Avoid keeping a possible heavy object in memory once we are done with it
|
2018-09-24 06:01:03 +00:00
|
|
|
while relation_item = (tree_array.shift || null_iid_pipelines.shift)
|
|
|
|
if nil_iid_pipeline?(relation_key, relation_item) && tree_array.any?
|
|
|
|
# Move pipelines with NULL IIDs to the end
|
|
|
|
# so they don't clash with existing IIDs.
|
|
|
|
null_iid_pipelines << relation_item
|
|
|
|
|
|
|
|
next
|
|
|
|
end
|
|
|
|
|
2017-09-05 15:24:57 +00:00
|
|
|
# The transaction at this level is less speedy than one single transaction
|
|
|
|
# But we can't have it in the upper level or GC won't get rid of the AR objects
|
|
|
|
# after we save the batch.
|
2017-09-03 18:51:50 +00:00
|
|
|
Project.transaction do
|
2017-09-03 18:01:14 +00:00
|
|
|
process_sub_relation(relation, relation_item)
|
2017-09-03 17:16:18 +00:00
|
|
|
|
2018-10-30 10:53:01 +00:00
|
|
|
# For every subrelation that hangs from Project, save the associated records altogether
|
2017-09-05 15:24:57 +00:00
|
|
|
# This effectively batches all records per subrelation item, only keeping those in memory
|
|
|
|
# We have to keep in mind that more batch granularity << Memory, but >> Slowness
|
2017-09-03 17:16:18 +00:00
|
|
|
if save
|
|
|
|
save_relation_hash([relation_item], relation_key)
|
2017-09-03 18:01:14 +00:00
|
|
|
tree_hash[relation_key].delete(relation_item)
|
2017-09-03 17:16:18 +00:00
|
|
|
end
|
2017-09-03 18:51:50 +00:00
|
|
|
end
|
2016-04-11 16:30:54 +00:00
|
|
|
end
|
2017-09-03 17:16:18 +00:00
|
|
|
|
|
|
|
tree_hash.delete(relation_key) if save
|
2016-04-11 16:30:54 +00:00
|
|
|
end
|
|
|
|
|
2017-09-03 18:01:14 +00:00
|
|
|
def process_sub_relation(relation, relation_item)
|
|
|
|
relation.values.flatten.each do |sub_relation|
|
|
|
|
# We just use author to get the user ID, do not attempt to create an instance.
|
|
|
|
next if sub_relation == :author
|
|
|
|
|
2017-09-06 09:11:02 +00:00
|
|
|
create_sub_relations(sub_relation, relation_item, save: false) if sub_relation.is_a?(Hash)
|
2017-09-03 18:01:14 +00:00
|
|
|
|
|
|
|
relation_hash, sub_relation = assign_relation_hash(relation_item, sub_relation)
|
|
|
|
relation_item[sub_relation.to_s] = create_relation(sub_relation, relation_hash) unless relation_hash.blank?
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2016-06-13 14:55:51 +00:00
|
|
|
def assign_relation_hash(relation_item, sub_relation)
|
|
|
|
if sub_relation.is_a?(Hash)
|
|
|
|
relation_hash = relation_item[sub_relation.keys.first.to_s]
|
|
|
|
sub_relation = sub_relation.keys.first
|
|
|
|
else
|
|
|
|
relation_hash = relation_item[sub_relation.to_s]
|
|
|
|
end
|
2018-01-11 16:34:01 +00:00
|
|
|
|
2016-06-14 08:20:47 +00:00
|
|
|
[relation_hash, sub_relation]
|
2016-06-13 14:55:51 +00:00
|
|
|
end
|
|
|
|
|
2016-03-10 17:43:57 +00:00
|
|
|
def create_relation(relation, relation_hash_list)
|
2016-06-02 08:59:54 +00:00
|
|
|
relation_array = [relation_hash_list].flatten.map do |relation_hash|
|
2017-09-05 19:06:27 +00:00
|
|
|
Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym,
|
2018-06-22 07:48:44 +00:00
|
|
|
relation_hash: relation_hash,
|
2016-06-01 16:03:51 +00:00
|
|
|
members_mapper: members_mapper,
|
2016-08-12 10:04:33 +00:00
|
|
|
user: @user,
|
2018-05-03 09:02:26 +00:00
|
|
|
project: @restored_project,
|
|
|
|
excluded_keys: excluded_keys_for_relation(relation))
|
2016-12-16 15:13:46 +00:00
|
|
|
end.compact
|
2016-06-02 08:59:54 +00:00
|
|
|
|
|
|
|
relation_hash_list.is_a?(Array) ? relation_array : relation_array.first
|
2016-03-09 15:21:02 +00:00
|
|
|
end
|
2016-09-29 11:15:18 +00:00
|
|
|
|
2018-03-29 13:08:31 +00:00
|
|
|
def reader
|
|
|
|
@reader ||= Gitlab::ImportExport::Reader.new(shared: @shared)
|
|
|
|
end
|
2018-05-03 09:02:26 +00:00
|
|
|
|
|
|
|
def excluded_keys_for_relation(relation)
|
2018-08-29 13:41:56 +00:00
|
|
|
reader.attributes_finder.find_excluded_keys(relation)
|
2018-05-03 09:02:26 +00:00
|
|
|
end
|
2018-09-24 06:01:03 +00:00
|
|
|
|
|
|
|
def nil_iid_pipeline?(relation_key, relation_item)
|
2018-12-05 14:39:15 +00:00
|
|
|
relation_key == 'ci_pipelines' && relation_item['iid'].nil?
|
2018-09-24 06:01:03 +00:00
|
|
|
end
|
2016-03-09 15:21:02 +00:00
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|