1c481b7aac
Add an index to the `store` column on `uploads`. This makes counting local uploads faster. Also, there is no longer need to check for objects with `store = NULL`. See https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/18557 --- ### Query plans Query: ```sql SELECT COUNT(*) FROM "uploads" WHERE ("uploads"."store" = ? OR "uploads"."store" IS NULL) ``` #### Without index ``` gitlabhq_production=# EXPLAIN ANALYZE SELECT uploads.* FROM uploads WHERE (uploads.store = 1 OR uploads.store IS NULL); QUERY PLAN --------------------------------------------------------------------------------------------------------------- Seq Scan on uploads (cost=0.00..601729.54 rows=578 width=272) (actual time=6.170..2308.256 rows=545 loops=1) Filter: ((store = 1) OR (store IS NULL)) Rows Removed by Filter: 4411957 Planning time: 38.652 ms Execution time: 2308.454 ms (5 rows) ``` #### Add index ``` gitlabhq_production=# create index uploads_tmp1 on uploads (store); CREATE INDEX ``` #### With index ``` gitlabhq_production=# EXPLAIN ANALYZE SELECT uploads.* FROM uploads WHERE (uploads.store = 1 OR uploads.store IS NULL); QUERY PLAN ------------------------------------------------------------------------------------------------------------------------------- Bitmap Heap Scan on uploads (cost=11.46..1238.88 rows=574 width=272) (actual time=0.155..0.577 rows=545 loops=1) Recheck Cond: ((store = 1) OR (store IS NULL)) Heap Blocks: exact=217 -> BitmapOr (cost=11.46..11.46 rows=574 width=0) (actual time=0.116..0.116 rows=0 loops=1) -> Bitmap Index Scan on uploads_tmp1 (cost=0.00..8.74 rows=574 width=0) (actual time=0.095..0.095 rows=545 loops=1) Index Cond: (store = 1) -> Bitmap Index Scan on uploads_tmp1 (cost=0.00..2.44 rows=1 width=0) (actual time=0.020..0.020 rows=0 loops=1) Index Cond: (store IS NULL) Planning time: 0.274 ms Execution time: 0.637 ms (10 rows) ``` Closes https://gitlab.com/gitlab-org/gitlab-ee/issues/6070
96 lines
2.1 KiB
Ruby
96 lines
2.1 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
class Upload < ActiveRecord::Base
|
|
# Upper limit for foreground checksum processing
|
|
CHECKSUM_THRESHOLD = 100.megabytes
|
|
|
|
belongs_to :model, polymorphic: true # rubocop:disable Cop/PolymorphicAssociations
|
|
|
|
validates :size, presence: true
|
|
validates :path, presence: true
|
|
validates :model, presence: true
|
|
validates :uploader, presence: true
|
|
|
|
scope :with_files_stored_locally, -> { where(store: ObjectStorage::Store::LOCAL) }
|
|
|
|
before_save :calculate_checksum!, if: :foreground_checksummable?
|
|
after_commit :schedule_checksum, if: :checksummable?
|
|
|
|
# as the FileUploader is not mounted, the default CarrierWave ActiveRecord
|
|
# hooks are not executed and the file will not be deleted
|
|
after_destroy :delete_file!, if: -> { uploader_class <= FileUploader }
|
|
|
|
def self.hexdigest(path)
|
|
Digest::SHA256.file(path).hexdigest
|
|
end
|
|
|
|
def absolute_path
|
|
raise ObjectStorage::RemoteStoreError, "Remote object has no absolute path." unless local?
|
|
return path unless relative_path?
|
|
|
|
uploader_class.absolute_path(self)
|
|
end
|
|
|
|
def calculate_checksum!
|
|
self.checksum = nil
|
|
return unless checksummable?
|
|
|
|
self.checksum = Digest::SHA256.file(absolute_path).hexdigest
|
|
end
|
|
|
|
def build_uploader(mounted_as = nil)
|
|
uploader_class.new(model, mounted_as || mount_point).tap do |uploader|
|
|
uploader.upload = self
|
|
uploader.retrieve_from_store!(identifier)
|
|
end
|
|
end
|
|
|
|
def exist?
|
|
File.exist?(absolute_path)
|
|
end
|
|
|
|
def uploader_context
|
|
{
|
|
identifier: identifier,
|
|
secret: secret
|
|
}.compact
|
|
end
|
|
|
|
def local?
|
|
store == ObjectStorage::Store::LOCAL
|
|
end
|
|
|
|
private
|
|
|
|
def delete_file!
|
|
build_uploader.remove!
|
|
end
|
|
|
|
def checksummable?
|
|
checksum.nil? && local? && exist?
|
|
end
|
|
|
|
def foreground_checksummable?
|
|
checksummable? && size <= CHECKSUM_THRESHOLD
|
|
end
|
|
|
|
def schedule_checksum
|
|
UploadChecksumWorker.perform_async(id)
|
|
end
|
|
|
|
def relative_path?
|
|
!path.start_with?('/')
|
|
end
|
|
|
|
def uploader_class
|
|
Object.const_get(uploader)
|
|
end
|
|
|
|
def identifier
|
|
File.basename(path)
|
|
end
|
|
|
|
def mount_point
|
|
super&.to_sym
|
|
end
|
|
end
|