Remove codebase dependencies from a BG migration
Specifically, `PopulateUntrackedUploads` and its spec.
This commit is contained in:
Michael Kozono
parent
975dc69ec6
commit
348c60d9be
4 changed files with 275 additions and 112 deletions
|
|
@ -5,11 +5,15 @@ module Gitlab
|
|||
# This class processes a batch of rows in `untracked_files_for_uploads` by
|
||||
# adding each file to the `uploads` table if it does not exist.
|
||||
class PopulateUntrackedUploads # rubocop:disable Metrics/ClassLength
|
||||
include PopulateUntrackedUploadsDependencies
|
||||
|
||||
# This class is responsible for producing the attributes necessary to
|
||||
# track an uploaded file in the `uploads` table.
|
||||
class UntrackedFile < ActiveRecord::Base # rubocop:disable Metrics/ClassLength, Metrics/LineLength
|
||||
self.table_name = 'untracked_files_for_uploads'
|
||||
|
||||
include PopulateUntrackedUploadsDependencies
|
||||
|
||||
# Ends with /:random_hex/:filename
|
||||
FILE_UPLOADER_PATH = %r{/\h+/[^/]+\z}
|
||||
FULL_PATH_CAPTURE = /\A(.+)#{FILE_UPLOADER_PATH}/
|
||||
|
|
@ -147,11 +151,6 @@ module Gitlab
|
|||
end
|
||||
end
|
||||
|
||||
# This class is used to query the `uploads` table.
|
||||
class Upload < ActiveRecord::Base
|
||||
self.table_name = 'uploads'
|
||||
end
|
||||
|
||||
def perform(start_id, end_id)
|
||||
return unless migrate?
|
||||
|
||||
|
|
@ -229,7 +228,7 @@ module Gitlab
|
|||
end
|
||||
|
||||
ids.each do |model_type, model_ids|
|
||||
model_class = Object.const_get(model_type)
|
||||
model_class = self.class.const_get(model_type)
|
||||
found_ids = model_class.where(id: model_ids.uniq).pluck(:id)
|
||||
deleted_ids = ids[model_type] - found_ids
|
||||
ids[model_type] = deleted_ids
|
||||
|
|
|
|||
|
|
@ -0,0 +1,59 @@
|
|||
# frozen_string_literal: true
|
||||
module Gitlab
|
||||
module BackgroundMigration
|
||||
module PopulateUntrackedUploadsDependencies
|
||||
# Avoid using application code
|
||||
class Upload < ActiveRecord::Base
|
||||
self.table_name = 'uploads'
|
||||
end
|
||||
|
||||
# Avoid using application code
|
||||
class Appearance < ActiveRecord::Base
|
||||
self.table_name = 'appearances'
|
||||
end
|
||||
|
||||
# Avoid using application code
|
||||
class Namespace < ActiveRecord::Base
|
||||
self.table_name = 'namespaces'
|
||||
end
|
||||
|
||||
# Avoid using application code
|
||||
class Note < ActiveRecord::Base
|
||||
self.table_name = 'notes'
|
||||
end
|
||||
|
||||
# Avoid using application code
|
||||
class User < ActiveRecord::Base
|
||||
self.table_name = 'users'
|
||||
end
|
||||
|
||||
# Since project Markdown upload paths don't contain the project ID, we have to find the
|
||||
# project by its full_path. Due to MySQL/PostgreSQL differences, and historical reasons,
|
||||
# the logic is somewhat complex, so I've mostly copied it in here.
|
||||
class Project < ActiveRecord::Base
|
||||
self.table_name = 'projects'
|
||||
|
||||
def self.find_by_full_path(path)
|
||||
binary = Gitlab::Database.mysql? ? 'BINARY' : ''
|
||||
order_sql = "(CASE WHEN #{binary} routes.path = #{connection.quote(path)} THEN 0 ELSE 1 END)"
|
||||
where_full_path_in(path).reorder(order_sql).take
|
||||
end
|
||||
|
||||
def self.where_full_path_in(path)
|
||||
cast_lower = Gitlab::Database.postgresql?
|
||||
|
||||
path = connection.quote(path)
|
||||
|
||||
where =
|
||||
if cast_lower
|
||||
"(LOWER(routes.path) = LOWER(#{path}))"
|
||||
else
|
||||
"(routes.path = #{path})"
|
||||
end
|
||||
|
||||
joins("INNER JOIN routes ON routes.source_id = projects.id AND routes.source_type = 'Project'").where(where)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -1,17 +1,19 @@
|
|||
require 'spec_helper'
|
||||
|
||||
# This migration is using UploadService, which sets uploads.secret that is only
|
||||
# added to the DB schema in 20180129193323. Since the test isn't isolated, we
|
||||
# just use the latest schema when testing this migration.
|
||||
# Ideally, the test should not use factories nor UploadService, and rely on the
|
||||
# `table` helper instead.
|
||||
describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq, :migration, schema: 20180129193323 do
|
||||
# Rollback DB to 10.5 (later than this was originally written for) because it still needs to work.
|
||||
describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq, :migration, schema: 20180208183958 do
|
||||
include TrackUntrackedUploadsHelpers
|
||||
|
||||
subject { described_class.new }
|
||||
|
||||
let!(:untracked_files_for_uploads) { described_class::UntrackedFile }
|
||||
let!(:uploads) { described_class::Upload }
|
||||
let!(:appearances) { table(:appearances) }
|
||||
let!(:namespaces) { table(:namespaces) }
|
||||
let!(:notes) { table(:notes) }
|
||||
let!(:projects) { table(:projects) }
|
||||
let!(:routes) { table(:routes) }
|
||||
let!(:untracked_files_for_uploads) { table(:untracked_files_for_uploads) }
|
||||
let!(:uploads) { table(:uploads) }
|
||||
let!(:users) { table(:users) }
|
||||
|
||||
before do
|
||||
ensure_temporary_tracking_table_exists
|
||||
|
|
@ -19,30 +21,30 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq, :migra
|
|||
end
|
||||
|
||||
context 'with untracked files and tracked files in untracked_files_for_uploads' do
|
||||
let!(:appearance) { create_or_update_appearance(logo: uploaded_file, header_logo: uploaded_file) }
|
||||
let!(:user1) { create(:user, :with_avatar) }
|
||||
let!(:user2) { create(:user, :with_avatar) }
|
||||
let!(:project1) { create(:project, :legacy_storage, :with_avatar) }
|
||||
let!(:project2) { create(:project, :legacy_storage, :with_avatar) }
|
||||
let!(:appearance) { create_or_update_appearance(logo: true, header_logo: true) }
|
||||
let!(:user1) { create_user(avatar: true) }
|
||||
let!(:user2) { create_user(avatar: true) }
|
||||
let!(:project1) { create_project(avatar: true) }
|
||||
let!(:project2) { create_project(avatar: true) }
|
||||
|
||||
before do
|
||||
UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload
|
||||
UploadService.new(project2, uploaded_file, FileUploader).execute # Markdown upload
|
||||
add_markdown_attachment(project1)
|
||||
add_markdown_attachment(project2)
|
||||
|
||||
# File records created by PrepareUntrackedUploads
|
||||
untracked_files_for_uploads.create!(path: appearance.uploads.first.path)
|
||||
untracked_files_for_uploads.create!(path: appearance.uploads.last.path)
|
||||
untracked_files_for_uploads.create!(path: user1.uploads.first.path)
|
||||
untracked_files_for_uploads.create!(path: user2.uploads.first.path)
|
||||
untracked_files_for_uploads.create!(path: project1.uploads.first.path)
|
||||
untracked_files_for_uploads.create!(path: project2.uploads.first.path)
|
||||
untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project1.full_path}/#{project1.uploads.last.path}")
|
||||
untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project2.full_path}/#{project2.uploads.last.path}")
|
||||
untracked_files_for_uploads.create!(path: get_uploads(appearance, 'Appearance').first.path)
|
||||
untracked_files_for_uploads.create!(path: get_uploads(appearance, 'Appearance').last.path)
|
||||
untracked_files_for_uploads.create!(path: get_uploads(user1, 'User').first.path)
|
||||
untracked_files_for_uploads.create!(path: get_uploads(user2, 'User').first.path)
|
||||
untracked_files_for_uploads.create!(path: get_uploads(project1, 'Project').first.path)
|
||||
untracked_files_for_uploads.create!(path: get_uploads(project2, 'Project').first.path)
|
||||
untracked_files_for_uploads.create!(path: "#{project_uploads_dir(project1).sub("#{TrackUntrackedUploadsHelpers::PUBLIC_DIR}/", '')}/#{get_uploads(project1, 'Project').last.path}")
|
||||
untracked_files_for_uploads.create!(path: "#{project_uploads_dir(project2).sub("#{TrackUntrackedUploadsHelpers::PUBLIC_DIR}/", '')}/#{get_uploads(project2, 'Project').last.path}")
|
||||
|
||||
# Untrack 4 files
|
||||
user2.uploads.delete_all
|
||||
project2.uploads.delete_all # 2 files: avatar and a Markdown upload
|
||||
appearance.uploads.where("path like '%header_logo%'").delete_all
|
||||
get_uploads(user2, 'User').delete_all
|
||||
get_uploads(project2, 'Project').delete_all # 2 files: avatar and a Markdown upload
|
||||
get_uploads(appearance, 'Appearance').where("path like '%header_logo%'").delete_all
|
||||
end
|
||||
|
||||
it 'adds untracked files to the uploads table' do
|
||||
|
|
@ -50,9 +52,9 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq, :migra
|
|||
subject.perform(1, untracked_files_for_uploads.reorder(:id).last.id)
|
||||
end.to change { uploads.count }.from(4).to(8)
|
||||
|
||||
expect(user2.uploads.count).to eq(1)
|
||||
expect(project2.uploads.count).to eq(2)
|
||||
expect(appearance.uploads.count).to eq(2)
|
||||
expect(get_uploads(user2, 'User').count).to eq(1)
|
||||
expect(get_uploads(project2, 'Project').count).to eq(2)
|
||||
expect(get_uploads(appearance, 'Appearance').count).to eq(2)
|
||||
end
|
||||
|
||||
it 'deletes rows after processing them' do
|
||||
|
|
@ -66,9 +68,9 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq, :migra
|
|||
it 'does not create duplicate uploads of already tracked files' do
|
||||
subject.perform(1, untracked_files_for_uploads.last.id)
|
||||
|
||||
expect(user1.uploads.count).to eq(1)
|
||||
expect(project1.uploads.count).to eq(2)
|
||||
expect(appearance.uploads.count).to eq(2)
|
||||
expect(get_uploads(user1, 'User').count).to eq(1)
|
||||
expect(get_uploads(project1, 'Project').count).to eq(2)
|
||||
expect(get_uploads(appearance, 'Appearance').count).to eq(2)
|
||||
end
|
||||
|
||||
it 'uses the start and end batch ids [only 1st half]' do
|
||||
|
|
@ -80,11 +82,11 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq, :migra
|
|||
subject.perform(start_id, end_id)
|
||||
end.to change { uploads.count }.from(4).to(6)
|
||||
|
||||
expect(user1.uploads.count).to eq(1)
|
||||
expect(user2.uploads.count).to eq(1)
|
||||
expect(appearance.uploads.count).to eq(2)
|
||||
expect(project1.uploads.count).to eq(2)
|
||||
expect(project2.uploads.count).to eq(0)
|
||||
expect(get_uploads(user1, 'User').count).to eq(1)
|
||||
expect(get_uploads(user2, 'User').count).to eq(1)
|
||||
expect(get_uploads(appearance, 'Appearance').count).to eq(2)
|
||||
expect(get_uploads(project1, 'Project').count).to eq(2)
|
||||
expect(get_uploads(project2, 'Project').count).to eq(0)
|
||||
|
||||
# Only 4 have been either confirmed or added to uploads
|
||||
expect(untracked_files_for_uploads.count).to eq(4)
|
||||
|
|
@ -99,11 +101,11 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq, :migra
|
|||
subject.perform(start_id, end_id)
|
||||
end.to change { uploads.count }.from(4).to(6)
|
||||
|
||||
expect(user1.uploads.count).to eq(1)
|
||||
expect(user2.uploads.count).to eq(0)
|
||||
expect(appearance.uploads.count).to eq(1)
|
||||
expect(project1.uploads.count).to eq(2)
|
||||
expect(project2.uploads.count).to eq(2)
|
||||
expect(get_uploads(user1, 'User').count).to eq(1)
|
||||
expect(get_uploads(user2, 'User').count).to eq(0)
|
||||
expect(get_uploads(appearance, 'Appearance').count).to eq(1)
|
||||
expect(get_uploads(project1, 'Project').count).to eq(2)
|
||||
expect(get_uploads(project2, 'Project').count).to eq(2)
|
||||
|
||||
# Only 4 have been either confirmed or added to uploads
|
||||
expect(untracked_files_for_uploads.count).to eq(4)
|
||||
|
|
@ -122,7 +124,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq, :migra
|
|||
end
|
||||
|
||||
it 'does not block a whole batch because of one bad path' do
|
||||
untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project2.full_path}/._7d37bf4c747916390e596744117d5d1a")
|
||||
untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{get_full_path(project2)}/._7d37bf4c747916390e596744117d5d1a")
|
||||
expect(untracked_files_for_uploads.count).to eq(9)
|
||||
expect(uploads.count).to eq(4)
|
||||
|
||||
|
|
@ -133,7 +135,7 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq, :migra
|
|||
end
|
||||
|
||||
it 'an unparseable path is shown in error output' do
|
||||
bad_path = "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project2.full_path}/._7d37bf4c747916390e596744117d5d1a"
|
||||
bad_path = "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{get_full_path(project2)}/._7d37bf4c747916390e596744117d5d1a"
|
||||
untracked_files_for_uploads.create!(path: bad_path)
|
||||
|
||||
expect(Rails.logger).to receive(:error).with(/Error parsing path "#{bad_path}":/)
|
||||
|
|
@ -152,103 +154,113 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :sidekiq, :migra
|
|||
|
||||
describe 'upload outcomes for each path pattern' do
|
||||
shared_examples_for 'non_markdown_file' do
|
||||
let!(:expected_upload_attrs) { model.uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') }
|
||||
let!(:expected_upload_attrs) { model_uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') }
|
||||
let!(:untracked_file) { untracked_files_for_uploads.create!(path: expected_upload_attrs['path']) }
|
||||
|
||||
before do
|
||||
model.uploads.delete_all
|
||||
model_uploads.delete_all
|
||||
end
|
||||
|
||||
it 'creates an Upload record' do
|
||||
expect do
|
||||
subject.perform(1, untracked_files_for_uploads.last.id)
|
||||
end.to change { model.reload.uploads.count }.from(0).to(1)
|
||||
end.to change { model_uploads.count }.from(0).to(1)
|
||||
|
||||
expect(model.uploads.first.attributes).to include(expected_upload_attrs)
|
||||
expect(model_uploads.first.attributes).to include(expected_upload_attrs)
|
||||
end
|
||||
end
|
||||
|
||||
context 'for an appearance logo file path' do
|
||||
let(:model) { create_or_update_appearance(logo: uploaded_file) }
|
||||
let(:model) { create_or_update_appearance(logo: true) }
|
||||
let(:model_uploads) { get_uploads(model, 'Appearance') }
|
||||
|
||||
it_behaves_like 'non_markdown_file'
|
||||
end
|
||||
|
||||
context 'for an appearance header_logo file path' do
|
||||
let(:model) { create_or_update_appearance(header_logo: uploaded_file) }
|
||||
let(:model) { create_or_update_appearance(header_logo: true) }
|
||||
let(:model_uploads) { get_uploads(model, 'Appearance') }
|
||||
|
||||
it_behaves_like 'non_markdown_file'
|
||||
end
|
||||
|
||||
context 'for a pre-Markdown Note attachment file path' do
|
||||
let(:model) { create(:note, :with_attachment) }
|
||||
let!(:expected_upload_attrs) { Upload.where(model_type: 'Note', model_id: model.id).first.attributes.slice('path', 'uploader', 'size', 'checksum') }
|
||||
let(:model) { create_note(attachment: true) }
|
||||
let!(:expected_upload_attrs) { get_uploads(model, 'Note').first.attributes.slice('path', 'uploader', 'size', 'checksum') }
|
||||
let!(:untracked_file) { untracked_files_for_uploads.create!(path: expected_upload_attrs['path']) }
|
||||
|
||||
before do
|
||||
Upload.where(model_type: 'Note', model_id: model.id).delete_all
|
||||
get_uploads(model, 'Note').delete_all
|
||||
end
|
||||
|
||||
# Can't use the shared example because Note doesn't have an `uploads` association
|
||||
it 'creates an Upload record' do
|
||||
expect do
|
||||
subject.perform(1, untracked_files_for_uploads.last.id)
|
||||
end.to change { Upload.where(model_type: 'Note', model_id: model.id).count }.from(0).to(1)
|
||||
end.to change { get_uploads(model, 'Note').count }.from(0).to(1)
|
||||
|
||||
expect(Upload.where(model_type: 'Note', model_id: model.id).first.attributes).to include(expected_upload_attrs)
|
||||
expect(get_uploads(model, 'Note').first.attributes).to include(expected_upload_attrs)
|
||||
end
|
||||
end
|
||||
|
||||
context 'for a user avatar file path' do
|
||||
let(:model) { create(:user, :with_avatar) }
|
||||
let(:model) { create_user(avatar: true) }
|
||||
let(:model_uploads) { get_uploads(model, 'User') }
|
||||
|
||||
it_behaves_like 'non_markdown_file'
|
||||
end
|
||||
|
||||
context 'for a group avatar file path' do
|
||||
let(:model) { create(:group, :with_avatar) }
|
||||
let(:model) { create_group(avatar: true) }
|
||||
let(:model_uploads) { get_uploads(model, 'Namespace') }
|
||||
|
||||
it_behaves_like 'non_markdown_file'
|
||||
end
|
||||
|
||||
context 'for a project avatar file path' do
|
||||
let(:model) { create(:project, :legacy_storage, :with_avatar) }
|
||||
let(:model) { create_project(avatar: true) }
|
||||
let(:model_uploads) { get_uploads(model, 'Project') }
|
||||
|
||||
it_behaves_like 'non_markdown_file'
|
||||
end
|
||||
|
||||
context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
|
||||
let(:model) { create(:project, :legacy_storage) }
|
||||
let(:model) { create_project }
|
||||
|
||||
before do
|
||||
# Upload the file
|
||||
UploadService.new(model, uploaded_file, FileUploader).execute
|
||||
add_markdown_attachment(model)
|
||||
|
||||
# Create the untracked_files_for_uploads record
|
||||
untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{model.full_path}/#{model.uploads.first.path}")
|
||||
untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{get_full_path(model)}/#{get_uploads(model, 'Project').first.path}")
|
||||
|
||||
# Save the expected upload attributes
|
||||
@expected_upload_attrs = model.reload.uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum')
|
||||
@expected_upload_attrs = get_uploads(model, 'Project').first.attributes.slice('path', 'uploader', 'size', 'checksum')
|
||||
|
||||
# Untrack the file
|
||||
model.reload.uploads.delete_all
|
||||
get_uploads(model, 'Project').delete_all
|
||||
end
|
||||
|
||||
it 'creates an Upload record' do
|
||||
expect do
|
||||
subject.perform(1, untracked_files_for_uploads.last.id)
|
||||
end.to change { model.reload.uploads.count }.from(0).to(1)
|
||||
end.to change { get_uploads(model, 'Project').count }.from(0).to(1)
|
||||
|
||||
expect(model.uploads.first.attributes).to include(@expected_upload_attrs)
|
||||
expect(get_uploads(model, 'Project').first.attributes).to include(@expected_upload_attrs)
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do
|
||||
# Rollback DB to 10.5 (later than this was originally written for) because it still needs to work.
|
||||
describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile, :migration, schema: 20180208183958 do
|
||||
include TrackUntrackedUploadsHelpers
|
||||
|
||||
let(:upload_class) { Gitlab::BackgroundMigration::PopulateUntrackedUploads::Upload }
|
||||
let!(:appearances) { table(:appearances) }
|
||||
let!(:namespaces) { table(:namespaces) }
|
||||
let!(:projects) { table(:projects) }
|
||||
let!(:routes) { table(:routes) }
|
||||
let!(:uploads) { table(:uploads) }
|
||||
|
||||
before(:all) do
|
||||
ensure_temporary_tracking_table_exists
|
||||
|
|
@ -299,10 +311,10 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do
|
|||
|
||||
context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
|
||||
it 'returns the file path relative to the project directory in uploads' do
|
||||
project = create(:project, :legacy_storage)
|
||||
project = create_project
|
||||
random_hex = SecureRandom.hex
|
||||
|
||||
assert_upload_path("/#{project.full_path}/#{random_hex}/Some file.jpg", "#{random_hex}/Some file.jpg")
|
||||
assert_upload_path("/#{get_full_path(project)}/#{random_hex}/Some file.jpg", "#{random_hex}/Some file.jpg")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -352,9 +364,9 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do
|
|||
|
||||
context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
|
||||
it 'returns FileUploader as a string' do
|
||||
project = create(:project, :legacy_storage)
|
||||
project = create_project
|
||||
|
||||
assert_uploader("/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg", 'FileUploader')
|
||||
assert_uploader("/#{get_full_path(project)}/#{SecureRandom.hex}/Some file.jpg", 'FileUploader')
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -404,9 +416,9 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do
|
|||
|
||||
context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
|
||||
it 'returns Project as a string' do
|
||||
project = create(:project, :legacy_storage)
|
||||
project = create_project
|
||||
|
||||
assert_model_type("/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg", 'Project')
|
||||
assert_model_type("/#{get_full_path(project)}/#{SecureRandom.hex}/Some file.jpg", 'Project')
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -456,54 +468,42 @@ describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do
|
|||
|
||||
context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
|
||||
it 'returns the ID as a string' do
|
||||
project = create(:project, :legacy_storage)
|
||||
project = create_project
|
||||
|
||||
assert_model_id("/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg", project.id)
|
||||
assert_model_id("/#{get_full_path(project)}/#{SecureRandom.hex}/Some file.jpg", project.id)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe '#file_size' do
|
||||
context 'for an appearance logo file path' do
|
||||
let(:appearance) { create_or_update_appearance(logo: uploaded_file) }
|
||||
let(:untracked_file) { described_class.create!(path: appearance.uploads.first.path) }
|
||||
let(:appearance) { create_or_update_appearance(logo: true) }
|
||||
let(:untracked_file) { described_class.create!(path: get_uploads(appearance, 'Appearance').first.path) }
|
||||
|
||||
it 'returns the file size' do
|
||||
expect(untracked_file.file_size).to eq(35255)
|
||||
end
|
||||
|
||||
it 'returns the same thing that CarrierWave would return' do
|
||||
expect(untracked_file.file_size).to eq(appearance.logo.size)
|
||||
expect(untracked_file.file_size).to eq(1062)
|
||||
end
|
||||
end
|
||||
|
||||
context 'for a project avatar file path' do
|
||||
let(:project) { create(:project, :legacy_storage, avatar: uploaded_file) }
|
||||
let(:untracked_file) { described_class.create!(path: project.uploads.first.path) }
|
||||
let(:project) { create_project(avatar: true) }
|
||||
let(:untracked_file) { described_class.create!(path: get_uploads(project, 'Project').first.path) }
|
||||
|
||||
it 'returns the file size' do
|
||||
expect(untracked_file.file_size).to eq(35255)
|
||||
end
|
||||
|
||||
it 'returns the same thing that CarrierWave would return' do
|
||||
expect(untracked_file.file_size).to eq(project.avatar.size)
|
||||
expect(untracked_file.file_size).to eq(1062)
|
||||
end
|
||||
end
|
||||
|
||||
context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
|
||||
let(:project) { create(:project, :legacy_storage) }
|
||||
let(:untracked_file) { create_untracked_file("/#{project.full_path}/#{project.uploads.first.path}") }
|
||||
let(:project) { create_project }
|
||||
let(:untracked_file) { create_untracked_file("/#{get_full_path(project)}/#{get_uploads(project, 'Project').first.path}") }
|
||||
|
||||
before do
|
||||
UploadService.new(project, uploaded_file, FileUploader).execute
|
||||
add_markdown_attachment(project)
|
||||
end
|
||||
|
||||
it 'returns the file size' do
|
||||
expect(untracked_file.file_size).to eq(35255)
|
||||
end
|
||||
|
||||
it 'returns the same thing that CarrierWave would return' do
|
||||
expect(untracked_file.file_size).to eq(project.uploads.first.size)
|
||||
expect(untracked_file.file_size).to eq(1062)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -1,16 +1,121 @@
|
|||
module TrackUntrackedUploadsHelpers
|
||||
def uploaded_file
|
||||
fixture_path = Rails.root.join('spec/fixtures/rails_sample.jpg')
|
||||
fixture_file_upload(fixture_path)
|
||||
PUBLIC_DIR = File.join(Rails.root, 'tmp', 'tests', 'public')
|
||||
UPLOADS_DIR = File.join(PUBLIC_DIR, 'uploads')
|
||||
SYSTEM_DIR = File.join(UPLOADS_DIR, '-', 'system')
|
||||
UPLOAD_FILENAME = 'image.png'.freeze
|
||||
FIXTURE_FILE_PATH = File.join(Rails.root, 'spec', 'fixtures', 'dk.png')
|
||||
FIXTURE_CHECKSUM = 'b804383982bb89b00e828e3f44c038cc991d3d1768009fc39ba8e2c081b9fb75'.freeze
|
||||
|
||||
def create_or_update_appearance(logo: false, header_logo: false)
|
||||
appearance = appearances.first_or_create(title: 'foo', description: 'bar', logo: (UPLOAD_FILENAME if logo), header_logo: (UPLOAD_FILENAME if header_logo))
|
||||
|
||||
add_upload(appearance, 'Appearance', 'logo', 'AttachmentUploader') if logo
|
||||
add_upload(appearance, 'Appearance', 'header_logo', 'AttachmentUploader') if header_logo
|
||||
|
||||
appearance
|
||||
end
|
||||
|
||||
def create_group(avatar: false)
|
||||
index = unique_index(:group)
|
||||
group = namespaces.create(name: "group#{index}", path: "group#{index}", avatar: (UPLOAD_FILENAME if avatar))
|
||||
|
||||
add_upload(group, 'Group', 'avatar', 'AvatarUploader') if avatar
|
||||
|
||||
group
|
||||
end
|
||||
|
||||
def create_note(attachment: false)
|
||||
note = notes.create(attachment: (UPLOAD_FILENAME if attachment))
|
||||
|
||||
add_upload(note, 'Note', 'attachment', 'AttachmentUploader') if attachment
|
||||
|
||||
note
|
||||
end
|
||||
|
||||
def create_project(avatar: false)
|
||||
group = create_group
|
||||
project = projects.create(namespace_id: group.id, path: "project#{unique_index(:project)}", avatar: (UPLOAD_FILENAME if avatar))
|
||||
routes.create(path: "#{group.path}/#{project.path}", source_id: project.id, source_type: 'Project') # so Project.find_by_full_path works
|
||||
|
||||
add_upload(project, 'Project', 'avatar', 'AvatarUploader') if avatar
|
||||
|
||||
project
|
||||
end
|
||||
|
||||
def create_user(avatar: false)
|
||||
user = users.create(email: "foo#{unique_index(:user)}@bar.com", avatar: (UPLOAD_FILENAME if avatar), projects_limit: 100)
|
||||
|
||||
add_upload(user, 'User', 'avatar', 'AvatarUploader') if avatar
|
||||
|
||||
user
|
||||
end
|
||||
|
||||
def unique_index(name = :unnamed)
|
||||
@unique_index ||= {}
|
||||
@unique_index[name] ||= 0
|
||||
@unique_index[name] += 1
|
||||
end
|
||||
|
||||
def add_upload(model, model_type, attachment_type, uploader)
|
||||
file_path = upload_file_path(model, model_type, attachment_type)
|
||||
path_relative_to_public = file_path.sub("#{PUBLIC_DIR}/", '')
|
||||
create_file(file_path)
|
||||
|
||||
uploads.create!(
|
||||
size: 1062,
|
||||
path: path_relative_to_public,
|
||||
model_id: model.id,
|
||||
model_type: model_type == 'Group' ? 'Namespace' : model_type,
|
||||
uploader: uploader,
|
||||
checksum: FIXTURE_CHECKSUM
|
||||
)
|
||||
end
|
||||
|
||||
def add_markdown_attachment(project)
|
||||
project_dir = project_uploads_dir(project)
|
||||
attachment_dir = File.join(project_dir, SecureRandom.hex)
|
||||
attachment_file_path = File.join(attachment_dir, UPLOAD_FILENAME)
|
||||
project_attachment_path_relative_to_project = attachment_file_path.sub("#{project_dir}/", '')
|
||||
create_file(attachment_file_path)
|
||||
|
||||
uploads.create!(
|
||||
size: 1062,
|
||||
path: project_attachment_path_relative_to_project,
|
||||
model_id: project.id,
|
||||
model_type: 'Project',
|
||||
uploader: 'FileUploader',
|
||||
checksum: FIXTURE_CHECKSUM
|
||||
)
|
||||
end
|
||||
|
||||
def project_uploads_dir(project)
|
||||
File.join(UPLOADS_DIR, project.full_path)
|
||||
end
|
||||
|
||||
def upload_file_path(model, model_type, attachment_type)
|
||||
dir = File.join(upload_dir(model_type.downcase, attachment_type.to_s), model.id.to_s)
|
||||
File.join(dir, UPLOAD_FILENAME)
|
||||
end
|
||||
|
||||
def upload_dir(model_type, attachment_type)
|
||||
File.join(SYSTEM_DIR, model_type, attachment_type)
|
||||
end
|
||||
|
||||
def create_file(path)
|
||||
File.delete(path) if File.exist?(path)
|
||||
FileUtils.mkdir_p(File.dirname(path))
|
||||
FileUtils.cp(FIXTURE_FILE_PATH, path)
|
||||
end
|
||||
|
||||
def get_uploads(model, model_type)
|
||||
uploads.where(model_type: model_type, model_id: model.id)
|
||||
end
|
||||
|
||||
def get_full_path(project)
|
||||
routes.find_by(source_id: project.id, source_type: 'Project').path
|
||||
end
|
||||
|
||||
def ensure_temporary_tracking_table_exists
|
||||
Gitlab::BackgroundMigration::PrepareUntrackedUploads.new.send(:ensure_temporary_tracking_table_exists)
|
||||
end
|
||||
|
||||
def create_or_update_appearance(attrs)
|
||||
a = Appearance.first_or_initialize(title: 'foo', description: 'bar')
|
||||
a.update!(attrs)
|
||||
a
|
||||
end
|
||||
end
|
||||
|
|
|
|||
Loading…
Reference in a new issue