2019-09-05 13:51:27 +00:00
|
|
|
# frozen_string_literal: true
|
|
|
|
|
|
|
|
require 'fast_spec_helper'
|
|
|
|
|
|
|
|
describe Gitlab::ImportExport::AttributesFinder do
|
|
|
|
describe '#find_root' do
|
|
|
|
subject { described_class.new(config: config).find_root(model_key) }
|
|
|
|
|
|
|
|
let(:test_config) { 'spec/support/import_export/import_export.yml' }
|
|
|
|
let(:config) { Gitlab::ImportExport::Config.new.to_h }
|
|
|
|
let(:model_key) { :project }
|
|
|
|
|
|
|
|
let(:project_tree_hash) do
|
|
|
|
{
|
|
|
|
except: [:id, :created_at],
|
|
|
|
include: [
|
|
|
|
{ issues: { include: [] } },
|
|
|
|
{ labels: { include: [] } },
|
|
|
|
{ merge_requests: {
|
|
|
|
except: [:iid],
|
|
|
|
include: [
|
|
|
|
{ merge_request_diff: {
|
Improve performance and memory usage of project export
ActiveModel::Serialization is simple in that it recursively calls
`as_json` on each object to serialize everything. However, for a model
like a Project, this can generate a query for every single association,
which can add up to tens of thousands of queries and lead to memory
bloat.
To improve this, we can do several things:
1. We use `tree:` and `preload:` to automatically generate
a list of all preloads that could be used to serialize
objects in bulk.
2. We observe that a single project has many issues, merge requests,
etc. Instead of serializing everything at once, which could lead to
database timeouts and high memory usage, we take each top-level
association and serialize the data in batches.
For example, we serialize the first 100 issues and preload all of
their associated events, notes, etc. before moving onto the next
batch. When we're done, we serialize merge requests in the same way.
We repeat this pattern for the remaining associations specified in
import_export.yml.
2019-09-09 15:40:49 +00:00
|
|
|
include: [],
|
|
|
|
preload: { source_project: nil }
|
2019-09-05 13:51:27 +00:00
|
|
|
} },
|
|
|
|
{ merge_request_test: { include: [] } }
|
|
|
|
],
|
Improve performance and memory usage of project export
ActiveModel::Serialization is simple in that it recursively calls
`as_json` on each object to serialize everything. However, for a model
like a Project, this can generate a query for every single association,
which can add up to tens of thousands of queries and lead to memory
bloat.
To improve this, we can do several things:
1. We use `tree:` and `preload:` to automatically generate
a list of all preloads that could be used to serialize
objects in bulk.
2. We observe that a single project has many issues, merge requests,
etc. Instead of serializing everything at once, which could lead to
database timeouts and high memory usage, we take each top-level
association and serialize the data in batches.
For example, we serialize the first 100 issues and preload all of
their associated events, notes, etc. before moving onto the next
batch. When we're done, we serialize merge requests in the same way.
We repeat this pattern for the remaining associations specified in
import_export.yml.
2019-09-09 15:40:49 +00:00
|
|
|
only: [:id],
|
|
|
|
preload: {
|
|
|
|
merge_request_diff: { source_project: nil },
|
|
|
|
merge_request_test: nil
|
|
|
|
}
|
2019-09-05 13:51:27 +00:00
|
|
|
} },
|
|
|
|
{ commit_statuses: {
|
Improve performance and memory usage of project export
ActiveModel::Serialization is simple in that it recursively calls
`as_json` on each object to serialize everything. However, for a model
like a Project, this can generate a query for every single association,
which can add up to tens of thousands of queries and lead to memory
bloat.
To improve this, we can do several things:
1. We use `tree:` and `preload:` to automatically generate
a list of all preloads that could be used to serialize
objects in bulk.
2. We observe that a single project has many issues, merge requests,
etc. Instead of serializing everything at once, which could lead to
database timeouts and high memory usage, we take each top-level
association and serialize the data in batches.
For example, we serialize the first 100 issues and preload all of
their associated events, notes, etc. before moving onto the next
batch. When we're done, we serialize merge requests in the same way.
We repeat this pattern for the remaining associations specified in
import_export.yml.
2019-09-09 15:40:49 +00:00
|
|
|
include: [{ commit: { include: [] } }],
|
|
|
|
preload: { commit: nil }
|
2019-09-05 13:51:27 +00:00
|
|
|
} },
|
|
|
|
{ project_members: {
|
|
|
|
include: [{ user: { include: [],
|
Improve performance and memory usage of project export
ActiveModel::Serialization is simple in that it recursively calls
`as_json` on each object to serialize everything. However, for a model
like a Project, this can generate a query for every single association,
which can add up to tens of thousands of queries and lead to memory
bloat.
To improve this, we can do several things:
1. We use `tree:` and `preload:` to automatically generate
a list of all preloads that could be used to serialize
objects in bulk.
2. We observe that a single project has many issues, merge requests,
etc. Instead of serializing everything at once, which could lead to
database timeouts and high memory usage, we take each top-level
association and serialize the data in batches.
For example, we serialize the first 100 issues and preload all of
their associated events, notes, etc. before moving onto the next
batch. When we're done, we serialize merge requests in the same way.
We repeat this pattern for the remaining associations specified in
import_export.yml.
2019-09-09 15:40:49 +00:00
|
|
|
only: [:email] } }],
|
|
|
|
preload: { user: nil }
|
2019-09-05 13:51:27 +00:00
|
|
|
} }
|
Improve performance and memory usage of project export
ActiveModel::Serialization is simple in that it recursively calls
`as_json` on each object to serialize everything. However, for a model
like a Project, this can generate a query for every single association,
which can add up to tens of thousands of queries and lead to memory
bloat.
To improve this, we can do several things:
1. We use `tree:` and `preload:` to automatically generate
a list of all preloads that could be used to serialize
objects in bulk.
2. We observe that a single project has many issues, merge requests,
etc. Instead of serializing everything at once, which could lead to
database timeouts and high memory usage, we take each top-level
association and serialize the data in batches.
For example, we serialize the first 100 issues and preload all of
their associated events, notes, etc. before moving onto the next
batch. When we're done, we serialize merge requests in the same way.
We repeat this pattern for the remaining associations specified in
import_export.yml.
2019-09-09 15:40:49 +00:00
|
|
|
],
|
|
|
|
preload: {
|
|
|
|
commit_statuses: {
|
|
|
|
commit: nil
|
|
|
|
},
|
|
|
|
issues: nil,
|
|
|
|
labels: nil,
|
|
|
|
merge_requests: {
|
|
|
|
merge_request_diff: { source_project: nil },
|
|
|
|
merge_request_test: nil
|
|
|
|
},
|
|
|
|
project_members: {
|
|
|
|
user: nil
|
|
|
|
}
|
|
|
|
}
|
2019-09-05 13:51:27 +00:00
|
|
|
}
|
|
|
|
end
|
|
|
|
|
|
|
|
before do
|
|
|
|
allow_any_instance_of(Gitlab::ImportExport).to receive(:config_file).and_return(test_config)
|
|
|
|
end
|
|
|
|
|
|
|
|
it 'generates hash from project tree config' do
|
|
|
|
is_expected.to match(project_tree_hash)
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'individual scenarios' do
|
|
|
|
it 'generates the correct hash for a single project relation' do
|
|
|
|
setup_yaml(tree: { project: [:issues] })
|
|
|
|
|
|
|
|
is_expected.to match(
|
Improve performance and memory usage of project export
ActiveModel::Serialization is simple in that it recursively calls
`as_json` on each object to serialize everything. However, for a model
like a Project, this can generate a query for every single association,
which can add up to tens of thousands of queries and lead to memory
bloat.
To improve this, we can do several things:
1. We use `tree:` and `preload:` to automatically generate
a list of all preloads that could be used to serialize
objects in bulk.
2. We observe that a single project has many issues, merge requests,
etc. Instead of serializing everything at once, which could lead to
database timeouts and high memory usage, we take each top-level
association and serialize the data in batches.
For example, we serialize the first 100 issues and preload all of
their associated events, notes, etc. before moving onto the next
batch. When we're done, we serialize merge requests in the same way.
We repeat this pattern for the remaining associations specified in
import_export.yml.
2019-09-09 15:40:49 +00:00
|
|
|
include: [{ issues: { include: [] } }],
|
|
|
|
preload: { issues: nil }
|
2019-09-05 13:51:27 +00:00
|
|
|
)
|
|
|
|
end
|
|
|
|
|
|
|
|
it 'generates the correct hash for a single project feature relation' do
|
|
|
|
setup_yaml(tree: { project: [:project_feature] })
|
|
|
|
|
|
|
|
is_expected.to match(
|
Improve performance and memory usage of project export
ActiveModel::Serialization is simple in that it recursively calls
`as_json` on each object to serialize everything. However, for a model
like a Project, this can generate a query for every single association,
which can add up to tens of thousands of queries and lead to memory
bloat.
To improve this, we can do several things:
1. We use `tree:` and `preload:` to automatically generate
a list of all preloads that could be used to serialize
objects in bulk.
2. We observe that a single project has many issues, merge requests,
etc. Instead of serializing everything at once, which could lead to
database timeouts and high memory usage, we take each top-level
association and serialize the data in batches.
For example, we serialize the first 100 issues and preload all of
their associated events, notes, etc. before moving onto the next
batch. When we're done, we serialize merge requests in the same way.
We repeat this pattern for the remaining associations specified in
import_export.yml.
2019-09-09 15:40:49 +00:00
|
|
|
include: [{ project_feature: { include: [] } }],
|
|
|
|
preload: { project_feature: nil }
|
2019-09-05 13:51:27 +00:00
|
|
|
)
|
|
|
|
end
|
|
|
|
|
|
|
|
it 'generates the correct hash for a multiple project relation' do
|
|
|
|
setup_yaml(tree: { project: [:issues, :snippets] })
|
|
|
|
|
|
|
|
is_expected.to match(
|
|
|
|
include: [{ issues: { include: [] } },
|
Improve performance and memory usage of project export
ActiveModel::Serialization is simple in that it recursively calls
`as_json` on each object to serialize everything. However, for a model
like a Project, this can generate a query for every single association,
which can add up to tens of thousands of queries and lead to memory
bloat.
To improve this, we can do several things:
1. We use `tree:` and `preload:` to automatically generate
a list of all preloads that could be used to serialize
objects in bulk.
2. We observe that a single project has many issues, merge requests,
etc. Instead of serializing everything at once, which could lead to
database timeouts and high memory usage, we take each top-level
association and serialize the data in batches.
For example, we serialize the first 100 issues and preload all of
their associated events, notes, etc. before moving onto the next
batch. When we're done, we serialize merge requests in the same way.
We repeat this pattern for the remaining associations specified in
import_export.yml.
2019-09-09 15:40:49 +00:00
|
|
|
{ snippets: { include: [] } }],
|
|
|
|
preload: { issues: nil, snippets: nil }
|
2019-09-05 13:51:27 +00:00
|
|
|
)
|
|
|
|
end
|
|
|
|
|
|
|
|
it 'generates the correct hash for a single sub-relation' do
|
|
|
|
setup_yaml(tree: { project: [issues: [:notes]] })
|
|
|
|
|
|
|
|
is_expected.to match(
|
Improve performance and memory usage of project export
ActiveModel::Serialization is simple in that it recursively calls
`as_json` on each object to serialize everything. However, for a model
like a Project, this can generate a query for every single association,
which can add up to tens of thousands of queries and lead to memory
bloat.
To improve this, we can do several things:
1. We use `tree:` and `preload:` to automatically generate
a list of all preloads that could be used to serialize
objects in bulk.
2. We observe that a single project has many issues, merge requests,
etc. Instead of serializing everything at once, which could lead to
database timeouts and high memory usage, we take each top-level
association and serialize the data in batches.
For example, we serialize the first 100 issues and preload all of
their associated events, notes, etc. before moving onto the next
batch. When we're done, we serialize merge requests in the same way.
We repeat this pattern for the remaining associations specified in
import_export.yml.
2019-09-09 15:40:49 +00:00
|
|
|
include: [{ issues: { include: [{ notes: { include: [] } }],
|
|
|
|
preload: { notes: nil } } }],
|
|
|
|
preload: { issues: { notes: nil } }
|
2019-09-05 13:51:27 +00:00
|
|
|
)
|
|
|
|
end
|
|
|
|
|
|
|
|
it 'generates the correct hash for a multiple sub-relation' do
|
|
|
|
setup_yaml(tree: { project: [merge_requests: [:notes, :merge_request_diff]] })
|
|
|
|
|
|
|
|
is_expected.to match(
|
|
|
|
include: [{ merge_requests:
|
|
|
|
{ include: [{ notes: { include: [] } },
|
Improve performance and memory usage of project export
ActiveModel::Serialization is simple in that it recursively calls
`as_json` on each object to serialize everything. However, for a model
like a Project, this can generate a query for every single association,
which can add up to tens of thousands of queries and lead to memory
bloat.
To improve this, we can do several things:
1. We use `tree:` and `preload:` to automatically generate
a list of all preloads that could be used to serialize
objects in bulk.
2. We observe that a single project has many issues, merge requests,
etc. Instead of serializing everything at once, which could lead to
database timeouts and high memory usage, we take each top-level
association and serialize the data in batches.
For example, we serialize the first 100 issues and preload all of
their associated events, notes, etc. before moving onto the next
batch. When we're done, we serialize merge requests in the same way.
We repeat this pattern for the remaining associations specified in
import_export.yml.
2019-09-09 15:40:49 +00:00
|
|
|
{ merge_request_diff: { include: [] } }],
|
|
|
|
preload: { merge_request_diff: nil, notes: nil } } }],
|
|
|
|
preload: { merge_requests: { merge_request_diff: nil, notes: nil } }
|
2019-09-05 13:51:27 +00:00
|
|
|
)
|
|
|
|
end
|
|
|
|
|
|
|
|
it 'generates the correct hash for a sub-relation with another sub-relation' do
|
|
|
|
setup_yaml(tree: { project: [merge_requests: [notes: [:author]]] })
|
|
|
|
|
|
|
|
is_expected.to match(
|
|
|
|
include: [{ merge_requests: {
|
Improve performance and memory usage of project export
ActiveModel::Serialization is simple in that it recursively calls
`as_json` on each object to serialize everything. However, for a model
like a Project, this can generate a query for every single association,
which can add up to tens of thousands of queries and lead to memory
bloat.
To improve this, we can do several things:
1. We use `tree:` and `preload:` to automatically generate
a list of all preloads that could be used to serialize
objects in bulk.
2. We observe that a single project has many issues, merge requests,
etc. Instead of serializing everything at once, which could lead to
database timeouts and high memory usage, we take each top-level
association and serialize the data in batches.
For example, we serialize the first 100 issues and preload all of
their associated events, notes, etc. before moving onto the next
batch. When we're done, we serialize merge requests in the same way.
We repeat this pattern for the remaining associations specified in
import_export.yml.
2019-09-09 15:40:49 +00:00
|
|
|
include: [{ notes: { include: [{ author: { include: [] } }],
|
|
|
|
preload: { author: nil } } }],
|
|
|
|
preload: { notes: { author: nil } }
|
|
|
|
} }],
|
|
|
|
preload: { merge_requests: { notes: { author: nil } } }
|
2019-09-05 13:51:27 +00:00
|
|
|
)
|
|
|
|
end
|
|
|
|
|
|
|
|
it 'generates the correct hash for a relation with included attributes' do
|
|
|
|
setup_yaml(tree: { project: [:issues] },
|
|
|
|
included_attributes: { issues: [:name, :description] })
|
|
|
|
|
|
|
|
is_expected.to match(
|
|
|
|
include: [{ issues: { include: [],
|
Improve performance and memory usage of project export
ActiveModel::Serialization is simple in that it recursively calls
`as_json` on each object to serialize everything. However, for a model
like a Project, this can generate a query for every single association,
which can add up to tens of thousands of queries and lead to memory
bloat.
To improve this, we can do several things:
1. We use `tree:` and `preload:` to automatically generate
a list of all preloads that could be used to serialize
objects in bulk.
2. We observe that a single project has many issues, merge requests,
etc. Instead of serializing everything at once, which could lead to
database timeouts and high memory usage, we take each top-level
association and serialize the data in batches.
For example, we serialize the first 100 issues and preload all of
their associated events, notes, etc. before moving onto the next
batch. When we're done, we serialize merge requests in the same way.
We repeat this pattern for the remaining associations specified in
import_export.yml.
2019-09-09 15:40:49 +00:00
|
|
|
only: [:name, :description] } }],
|
|
|
|
preload: { issues: nil }
|
2019-09-05 13:51:27 +00:00
|
|
|
)
|
|
|
|
end
|
|
|
|
|
|
|
|
it 'generates the correct hash for a relation with excluded attributes' do
|
|
|
|
setup_yaml(tree: { project: [:issues] },
|
|
|
|
excluded_attributes: { issues: [:name] })
|
|
|
|
|
|
|
|
is_expected.to match(
|
|
|
|
include: [{ issues: { except: [:name],
|
Improve performance and memory usage of project export
ActiveModel::Serialization is simple in that it recursively calls
`as_json` on each object to serialize everything. However, for a model
like a Project, this can generate a query for every single association,
which can add up to tens of thousands of queries and lead to memory
bloat.
To improve this, we can do several things:
1. We use `tree:` and `preload:` to automatically generate
a list of all preloads that could be used to serialize
objects in bulk.
2. We observe that a single project has many issues, merge requests,
etc. Instead of serializing everything at once, which could lead to
database timeouts and high memory usage, we take each top-level
association and serialize the data in batches.
For example, we serialize the first 100 issues and preload all of
their associated events, notes, etc. before moving onto the next
batch. When we're done, we serialize merge requests in the same way.
We repeat this pattern for the remaining associations specified in
import_export.yml.
2019-09-09 15:40:49 +00:00
|
|
|
include: [] } }],
|
|
|
|
preload: { issues: nil }
|
2019-09-05 13:51:27 +00:00
|
|
|
)
|
|
|
|
end
|
|
|
|
|
|
|
|
it 'generates the correct hash for a relation with both excluded and included attributes' do
|
|
|
|
setup_yaml(tree: { project: [:issues] },
|
|
|
|
excluded_attributes: { issues: [:name] },
|
|
|
|
included_attributes: { issues: [:description] })
|
|
|
|
|
|
|
|
is_expected.to match(
|
|
|
|
include: [{ issues: { except: [:name],
|
|
|
|
include: [],
|
Improve performance and memory usage of project export
ActiveModel::Serialization is simple in that it recursively calls
`as_json` on each object to serialize everything. However, for a model
like a Project, this can generate a query for every single association,
which can add up to tens of thousands of queries and lead to memory
bloat.
To improve this, we can do several things:
1. We use `tree:` and `preload:` to automatically generate
a list of all preloads that could be used to serialize
objects in bulk.
2. We observe that a single project has many issues, merge requests,
etc. Instead of serializing everything at once, which could lead to
database timeouts and high memory usage, we take each top-level
association and serialize the data in batches.
For example, we serialize the first 100 issues and preload all of
their associated events, notes, etc. before moving onto the next
batch. When we're done, we serialize merge requests in the same way.
We repeat this pattern for the remaining associations specified in
import_export.yml.
2019-09-09 15:40:49 +00:00
|
|
|
only: [:description] } }],
|
|
|
|
preload: { issues: nil }
|
2019-09-05 13:51:27 +00:00
|
|
|
)
|
|
|
|
end
|
|
|
|
|
|
|
|
it 'generates the correct hash for a relation with custom methods' do
|
|
|
|
setup_yaml(tree: { project: [:issues] },
|
|
|
|
methods: { issues: [:name] })
|
|
|
|
|
|
|
|
is_expected.to match(
|
|
|
|
include: [{ issues: { include: [],
|
Improve performance and memory usage of project export
ActiveModel::Serialization is simple in that it recursively calls
`as_json` on each object to serialize everything. However, for a model
like a Project, this can generate a query for every single association,
which can add up to tens of thousands of queries and lead to memory
bloat.
To improve this, we can do several things:
1. We use `tree:` and `preload:` to automatically generate
a list of all preloads that could be used to serialize
objects in bulk.
2. We observe that a single project has many issues, merge requests,
etc. Instead of serializing everything at once, which could lead to
database timeouts and high memory usage, we take each top-level
association and serialize the data in batches.
For example, we serialize the first 100 issues and preload all of
their associated events, notes, etc. before moving onto the next
batch. When we're done, we serialize merge requests in the same way.
We repeat this pattern for the remaining associations specified in
import_export.yml.
2019-09-09 15:40:49 +00:00
|
|
|
methods: [:name] } }],
|
|
|
|
preload: { issues: nil }
|
2019-09-05 13:51:27 +00:00
|
|
|
)
|
|
|
|
end
|
|
|
|
|
|
|
|
def setup_yaml(hash)
|
|
|
|
allow(YAML).to receive(:load_file).with(test_config).and_return(hash)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
describe '#find_relations_tree' do
|
|
|
|
subject { described_class.new(config: config).find_relations_tree(model_key) }
|
|
|
|
|
|
|
|
let(:tree) { { project: { issues: {} } } }
|
|
|
|
let(:model_key) { :project }
|
|
|
|
|
|
|
|
context 'when initialized with config including tree' do
|
|
|
|
let(:config) { { tree: tree } }
|
|
|
|
|
|
|
|
context 'when relation is in top-level keys of the tree' do
|
|
|
|
it { is_expected.to eq({ issues: {} }) }
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'when the relation is not in top-level keys' do
|
|
|
|
let(:model_key) { :issues }
|
|
|
|
|
|
|
|
it { is_expected.to be_nil }
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'when tree is not present in config' do
|
|
|
|
let(:config) { {} }
|
|
|
|
|
|
|
|
it { is_expected.to be_nil }
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
describe '#find_excluded_keys' do
|
|
|
|
subject { described_class.new(config: config).find_excluded_keys(klass_name) }
|
|
|
|
|
|
|
|
let(:klass_name) { 'project' }
|
|
|
|
|
|
|
|
context 'when initialized with excluded_attributes' do
|
|
|
|
let(:config) { { excluded_attributes: excluded_attributes } }
|
|
|
|
let(:excluded_attributes) { { project: [:name, :path], issues: [:milestone_id] } }
|
|
|
|
|
|
|
|
it { is_expected.to eq(%w[name path]) }
|
|
|
|
end
|
|
|
|
|
|
|
|
context 'when excluded_attributes are not present in config' do
|
|
|
|
let(:config) { {} }
|
|
|
|
|
|
|
|
it { is_expected.to eq([]) }
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|