gitlab-org--gitlab-foss/spec/models/upload_spec.rb

216 lines
6.5 KiB
Ruby
Raw Normal View History

# frozen_string_literal: true
require 'spec_helper'
describe Upload do
describe 'associations' do
it { is_expected.to belong_to(:model) }
end
describe 'validations' do
it { is_expected.to validate_presence_of(:size) }
it { is_expected.to validate_presence_of(:path) }
it { is_expected.to validate_presence_of(:model) }
it { is_expected.to validate_presence_of(:uploader) }
end
describe 'callbacks' do
context 'for a file above the checksum threshold' do
it 'schedules checksum calculation' do
stub_const('UploadChecksumWorker', spy)
upload = described_class.create(
path: __FILE__,
size: described_class::CHECKSUM_THRESHOLD + 1.kilobyte,
model: build_stubbed(:user),
Enhance performance of counting local Uploads Add an index to the `store` column on `uploads`. This makes counting local uploads faster. Also, there is no longer need to check for objects with `store = NULL`. See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/18557 --- ### Query plans Query: ```sql SELECT COUNT(*) FROM "uploads" WHERE ("uploads"."store" = ? OR "uploads"."store" IS NULL) ``` #### Without index ``` gitlabhq_production=# EXPLAIN ANALYZE SELECT uploads.* FROM uploads WHERE (uploads.store = 1 OR uploads.store IS NULL); QUERY PLAN --------------------------------------------------------------------------------------------------------------- Seq Scan on uploads (cost=0.00..601729.54 rows=578 width=272) (actual time=6.170..2308.256 rows=545 loops=1) Filter: ((store = 1) OR (store IS NULL)) Rows Removed by Filter: 4411957 Planning time: 38.652 ms Execution time: 2308.454 ms (5 rows) ``` #### Add index ``` gitlabhq_production=# create index uploads_tmp1 on uploads (store); CREATE INDEX ``` #### With index ``` gitlabhq_production=# EXPLAIN ANALYZE SELECT uploads.* FROM uploads WHERE (uploads.store = 1 OR uploads.store IS NULL); QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------- Bitmap Heap Scan on uploads (cost=11.46..1238.88 rows=574 width=272) (actual time=0.155..0.577 rows=545 loops=1) Recheck Cond: ((store = 1) OR (store IS NULL)) Heap Blocks: exact=217 -> BitmapOr (cost=11.46..11.46 rows=574 width=0) (actual time=0.116..0.116 rows=0 loops=1) -> Bitmap Index Scan on uploads_tmp1 (cost=0.00..8.74 rows=574 width=0) (actual time=0.095..0.095 rows=545 loops=1) Index Cond: (store = 1) -> Bitmap Index Scan on uploads_tmp1 (cost=0.00..2.44 rows=1 width=0) (actual time=0.020..0.020 rows=0 loops=1) Index Cond: (store IS NULL) Planning time: 0.274 ms Execution time: 0.637 ms (10 rows) ``` Closes https://gitlab.com/gitlab-org/gitlab-ee/issues/6070
2018-10-05 09:59:58 -04:00
uploader: double('ExampleUploader'),
store: ObjectStorage::Store::LOCAL
)
expect(UploadChecksumWorker)
.to have_received(:perform_async).with(upload.id)
end
end
context 'for a file at or below the checksum threshold' do
it 'calculates checksum immediately before save' do
upload = described_class.new(
path: __FILE__,
size: described_class::CHECKSUM_THRESHOLD,
model: build_stubbed(:user),
Enhance performance of counting local Uploads Add an index to the `store` column on `uploads`. This makes counting local uploads faster. Also, there is no longer need to check for objects with `store = NULL`. See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/18557 --- ### Query plans Query: ```sql SELECT COUNT(*) FROM "uploads" WHERE ("uploads"."store" = ? OR "uploads"."store" IS NULL) ``` #### Without index ``` gitlabhq_production=# EXPLAIN ANALYZE SELECT uploads.* FROM uploads WHERE (uploads.store = 1 OR uploads.store IS NULL); QUERY PLAN --------------------------------------------------------------------------------------------------------------- Seq Scan on uploads (cost=0.00..601729.54 rows=578 width=272) (actual time=6.170..2308.256 rows=545 loops=1) Filter: ((store = 1) OR (store IS NULL)) Rows Removed by Filter: 4411957 Planning time: 38.652 ms Execution time: 2308.454 ms (5 rows) ``` #### Add index ``` gitlabhq_production=# create index uploads_tmp1 on uploads (store); CREATE INDEX ``` #### With index ``` gitlabhq_production=# EXPLAIN ANALYZE SELECT uploads.* FROM uploads WHERE (uploads.store = 1 OR uploads.store IS NULL); QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------- Bitmap Heap Scan on uploads (cost=11.46..1238.88 rows=574 width=272) (actual time=0.155..0.577 rows=545 loops=1) Recheck Cond: ((store = 1) OR (store IS NULL)) Heap Blocks: exact=217 -> BitmapOr (cost=11.46..11.46 rows=574 width=0) (actual time=0.116..0.116 rows=0 loops=1) -> Bitmap Index Scan on uploads_tmp1 (cost=0.00..8.74 rows=574 width=0) (actual time=0.095..0.095 rows=545 loops=1) Index Cond: (store = 1) -> Bitmap Index Scan on uploads_tmp1 (cost=0.00..2.44 rows=1 width=0) (actual time=0.020..0.020 rows=0 loops=1) Index Cond: (store IS NULL) Planning time: 0.274 ms Execution time: 0.637 ms (10 rows) ``` Closes https://gitlab.com/gitlab-org/gitlab-ee/issues/6070
2018-10-05 09:59:58 -04:00
uploader: double('ExampleUploader'),
store: ObjectStorage::Store::LOCAL
)
expect { upload.save }
.to change { upload.checksum }.from(nil)
.to(a_string_matching(/\A\h{64}\z/))
end
end
2018-01-31 10:59:35 -05:00
describe 'after_destroy' do
context 'uploader is FileUploader-based' do
subject { create(:upload, :issuable_upload) }
it 'calls delete_file!' do
is_expected.to receive(:delete_file!)
subject.destroy
end
end
end
end
describe '#absolute_path' do
it 'returns the path directly when already absolute' do
path = '/path/to/namespace/project/secret/file.jpg'
Enhance performance of counting local Uploads Add an index to the `store` column on `uploads`. This makes counting local uploads faster. Also, there is no longer need to check for objects with `store = NULL`. See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/18557 --- ### Query plans Query: ```sql SELECT COUNT(*) FROM "uploads" WHERE ("uploads"."store" = ? OR "uploads"."store" IS NULL) ``` #### Without index ``` gitlabhq_production=# EXPLAIN ANALYZE SELECT uploads.* FROM uploads WHERE (uploads.store = 1 OR uploads.store IS NULL); QUERY PLAN --------------------------------------------------------------------------------------------------------------- Seq Scan on uploads (cost=0.00..601729.54 rows=578 width=272) (actual time=6.170..2308.256 rows=545 loops=1) Filter: ((store = 1) OR (store IS NULL)) Rows Removed by Filter: 4411957 Planning time: 38.652 ms Execution time: 2308.454 ms (5 rows) ``` #### Add index ``` gitlabhq_production=# create index uploads_tmp1 on uploads (store); CREATE INDEX ``` #### With index ``` gitlabhq_production=# EXPLAIN ANALYZE SELECT uploads.* FROM uploads WHERE (uploads.store = 1 OR uploads.store IS NULL); QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------- Bitmap Heap Scan on uploads (cost=11.46..1238.88 rows=574 width=272) (actual time=0.155..0.577 rows=545 loops=1) Recheck Cond: ((store = 1) OR (store IS NULL)) Heap Blocks: exact=217 -> BitmapOr (cost=11.46..11.46 rows=574 width=0) (actual time=0.116..0.116 rows=0 loops=1) -> Bitmap Index Scan on uploads_tmp1 (cost=0.00..8.74 rows=574 width=0) (actual time=0.095..0.095 rows=545 loops=1) Index Cond: (store = 1) -> Bitmap Index Scan on uploads_tmp1 (cost=0.00..2.44 rows=1 width=0) (actual time=0.020..0.020 rows=0 loops=1) Index Cond: (store IS NULL) Planning time: 0.274 ms Execution time: 0.637 ms (10 rows) ``` Closes https://gitlab.com/gitlab-org/gitlab-ee/issues/6070
2018-10-05 09:59:58 -04:00
upload = described_class.new(path: path, store: ObjectStorage::Store::LOCAL)
expect(upload).not_to receive(:uploader_class)
expect(upload.absolute_path).to eq path
end
it "delegates to the uploader's absolute_path method" do
uploader = spy('FakeUploader')
Enhance performance of counting local Uploads Add an index to the `store` column on `uploads`. This makes counting local uploads faster. Also, there is no longer need to check for objects with `store = NULL`. See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/18557 --- ### Query plans Query: ```sql SELECT COUNT(*) FROM "uploads" WHERE ("uploads"."store" = ? OR "uploads"."store" IS NULL) ``` #### Without index ``` gitlabhq_production=# EXPLAIN ANALYZE SELECT uploads.* FROM uploads WHERE (uploads.store = 1 OR uploads.store IS NULL); QUERY PLAN --------------------------------------------------------------------------------------------------------------- Seq Scan on uploads (cost=0.00..601729.54 rows=578 width=272) (actual time=6.170..2308.256 rows=545 loops=1) Filter: ((store = 1) OR (store IS NULL)) Rows Removed by Filter: 4411957 Planning time: 38.652 ms Execution time: 2308.454 ms (5 rows) ``` #### Add index ``` gitlabhq_production=# create index uploads_tmp1 on uploads (store); CREATE INDEX ``` #### With index ``` gitlabhq_production=# EXPLAIN ANALYZE SELECT uploads.* FROM uploads WHERE (uploads.store = 1 OR uploads.store IS NULL); QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------- Bitmap Heap Scan on uploads (cost=11.46..1238.88 rows=574 width=272) (actual time=0.155..0.577 rows=545 loops=1) Recheck Cond: ((store = 1) OR (store IS NULL)) Heap Blocks: exact=217 -> BitmapOr (cost=11.46..11.46 rows=574 width=0) (actual time=0.116..0.116 rows=0 loops=1) -> Bitmap Index Scan on uploads_tmp1 (cost=0.00..8.74 rows=574 width=0) (actual time=0.095..0.095 rows=545 loops=1) Index Cond: (store = 1) -> Bitmap Index Scan on uploads_tmp1 (cost=0.00..2.44 rows=1 width=0) (actual time=0.020..0.020 rows=0 loops=1) Index Cond: (store IS NULL) Planning time: 0.274 ms Execution time: 0.637 ms (10 rows) ``` Closes https://gitlab.com/gitlab-org/gitlab-ee/issues/6070
2018-10-05 09:59:58 -04:00
upload = described_class.new(path: 'secret/file.jpg', store: ObjectStorage::Store::LOCAL)
expect(upload).to receive(:uploader_class).and_return(uploader)
upload.absolute_path
expect(uploader).to have_received(:absolute_path).with(upload)
end
end
2018-01-29 12:57:34 -05:00
describe '#calculate_checksum!' do
let(:upload) do
described_class.new(path: __FILE__,
Enhance performance of counting local Uploads Add an index to the `store` column on `uploads`. This makes counting local uploads faster. Also, there is no longer need to check for objects with `store = NULL`. See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/18557 --- ### Query plans Query: ```sql SELECT COUNT(*) FROM "uploads" WHERE ("uploads"."store" = ? OR "uploads"."store" IS NULL) ``` #### Without index ``` gitlabhq_production=# EXPLAIN ANALYZE SELECT uploads.* FROM uploads WHERE (uploads.store = 1 OR uploads.store IS NULL); QUERY PLAN --------------------------------------------------------------------------------------------------------------- Seq Scan on uploads (cost=0.00..601729.54 rows=578 width=272) (actual time=6.170..2308.256 rows=545 loops=1) Filter: ((store = 1) OR (store IS NULL)) Rows Removed by Filter: 4411957 Planning time: 38.652 ms Execution time: 2308.454 ms (5 rows) ``` #### Add index ``` gitlabhq_production=# create index uploads_tmp1 on uploads (store); CREATE INDEX ``` #### With index ``` gitlabhq_production=# EXPLAIN ANALYZE SELECT uploads.* FROM uploads WHERE (uploads.store = 1 OR uploads.store IS NULL); QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------- Bitmap Heap Scan on uploads (cost=11.46..1238.88 rows=574 width=272) (actual time=0.155..0.577 rows=545 loops=1) Recheck Cond: ((store = 1) OR (store IS NULL)) Heap Blocks: exact=217 -> BitmapOr (cost=11.46..11.46 rows=574 width=0) (actual time=0.116..0.116 rows=0 loops=1) -> Bitmap Index Scan on uploads_tmp1 (cost=0.00..8.74 rows=574 width=0) (actual time=0.095..0.095 rows=545 loops=1) Index Cond: (store = 1) -> Bitmap Index Scan on uploads_tmp1 (cost=0.00..2.44 rows=1 width=0) (actual time=0.020..0.020 rows=0 loops=1) Index Cond: (store IS NULL) Planning time: 0.274 ms Execution time: 0.637 ms (10 rows) ``` Closes https://gitlab.com/gitlab-org/gitlab-ee/issues/6070
2018-10-05 09:59:58 -04:00
size: described_class::CHECKSUM_THRESHOLD - 1.megabyte,
store: ObjectStorage::Store::LOCAL)
2018-01-29 12:57:34 -05:00
end
it 'sets `checksum` to SHA256 sum of the file' do
expected = Digest::SHA256.file(__FILE__).hexdigest
2018-01-29 12:57:34 -05:00
expect { upload.calculate_checksum! }
.to change { upload.checksum }.from(nil).to(expected)
end
2018-10-30 06:53:01 -04:00
it 'sets `checksum` to nil for a non-existent file' do
expect(upload).to receive(:exist?).and_return(false)
2018-01-29 12:57:34 -05:00
checksum = Digest::SHA256.file(__FILE__).hexdigest
upload.checksum = checksum
expect { upload.calculate_checksum! }
.to change { upload.checksum }.from(checksum).to(nil)
end
end
describe '#build_uploader' do
it 'returns a uploader object with current upload associated with it' do
subject = build(:upload)
uploader = subject.build_uploader
expect(uploader.upload).to eq(subject)
expect(uploader.mounted_as).to eq(subject.send(:mount_point))
expect(uploader.file).to be_nil
end
end
describe '#retrieve_uploader' do
it 'returns a uploader object with current uploader associated with and cache retrieved' do
subject = build(:upload)
uploader = subject.retrieve_uploader
expect(uploader.upload).to eq(subject)
expect(uploader.mounted_as).to eq(subject.send(:mount_point))
expect(uploader.file).not_to be_nil
end
end
describe '#needs_checksum?' do
context 'with local storage' do
it 'returns true when no checksum exists' do
subject = create(:upload, :with_file, checksum: nil)
expect(subject.needs_checksum?).to be_truthy
end
it 'returns false when checksum is already present' do
subject = create(:upload, :with_file, checksum: 'something')
expect(subject.needs_checksum?).to be_falsey
end
end
context 'with remote storage' do
subject { build(:upload, :object_storage) }
it 'returns false' do
expect(subject.needs_checksum?).to be_falsey
end
end
end
describe '#exist?' do
it 'returns true when the file exists' do
Enhance performance of counting local Uploads Add an index to the `store` column on `uploads`. This makes counting local uploads faster. Also, there is no longer need to check for objects with `store = NULL`. See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/18557 --- ### Query plans Query: ```sql SELECT COUNT(*) FROM "uploads" WHERE ("uploads"."store" = ? OR "uploads"."store" IS NULL) ``` #### Without index ``` gitlabhq_production=# EXPLAIN ANALYZE SELECT uploads.* FROM uploads WHERE (uploads.store = 1 OR uploads.store IS NULL); QUERY PLAN --------------------------------------------------------------------------------------------------------------- Seq Scan on uploads (cost=0.00..601729.54 rows=578 width=272) (actual time=6.170..2308.256 rows=545 loops=1) Filter: ((store = 1) OR (store IS NULL)) Rows Removed by Filter: 4411957 Planning time: 38.652 ms Execution time: 2308.454 ms (5 rows) ``` #### Add index ``` gitlabhq_production=# create index uploads_tmp1 on uploads (store); CREATE INDEX ``` #### With index ``` gitlabhq_production=# EXPLAIN ANALYZE SELECT uploads.* FROM uploads WHERE (uploads.store = 1 OR uploads.store IS NULL); QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------- Bitmap Heap Scan on uploads (cost=11.46..1238.88 rows=574 width=272) (actual time=0.155..0.577 rows=545 loops=1) Recheck Cond: ((store = 1) OR (store IS NULL)) Heap Blocks: exact=217 -> BitmapOr (cost=11.46..11.46 rows=574 width=0) (actual time=0.116..0.116 rows=0 loops=1) -> Bitmap Index Scan on uploads_tmp1 (cost=0.00..8.74 rows=574 width=0) (actual time=0.095..0.095 rows=545 loops=1) Index Cond: (store = 1) -> Bitmap Index Scan on uploads_tmp1 (cost=0.00..2.44 rows=1 width=0) (actual time=0.020..0.020 rows=0 loops=1) Index Cond: (store IS NULL) Planning time: 0.274 ms Execution time: 0.637 ms (10 rows) ``` Closes https://gitlab.com/gitlab-org/gitlab-ee/issues/6070
2018-10-05 09:59:58 -04:00
upload = described_class.new(path: __FILE__, store: ObjectStorage::Store::LOCAL)
expect(upload).to exist
end
context 'when the file does not exist' do
it 'returns false' do
upload = described_class.new(path: "#{__FILE__}-nope", store: ObjectStorage::Store::LOCAL)
expect(upload).not_to exist
end
context 'when the record is persisted' do
it 'sends a message to Sentry' do
upload = create(:upload, :issuable_upload)
expect(Gitlab::ErrorTracking).to receive(:track_exception).with(instance_of(RuntimeError), upload.attributes)
upload.exist?
end
it 'increments a metric counter to signal a problem' do
upload = create(:upload, :issuable_upload)
counter = double(:counter)
expect(counter).to receive(:increment)
expect(Gitlab::Metrics).to receive(:counter).with(:upload_file_does_not_exist_total, 'The number of times an upload record could not find its file').and_return(counter)
upload.exist?
end
end
context 'when the record is not persisted' do
it 'does not send a message to Sentry' do
upload = described_class.new(path: "#{__FILE__}-nope", store: ObjectStorage::Store::LOCAL)
expect(Raven).not_to receive(:capture_message)
upload.exist?
end
it 'does not increment a metric counter' do
upload = described_class.new(path: "#{__FILE__}-nope", store: ObjectStorage::Store::LOCAL)
expect(Gitlab::Metrics).not_to receive(:counter)
upload.exist?
end
end
end
end
describe "#uploader_context" do
subject { create(:upload, :issuable_upload, secret: 'secret', filename: 'file.txt') }
it { expect(subject.uploader_context).to match(a_hash_including(secret: 'secret', identifier: 'file.txt')) }
end
end