gitlab-org--gitlab-foss/spec/lib/gitlab/background_migration/populate_untracked_uploads_spec.rb

require 'spec_helper'

describe Gitlab::BackgroundMigration::PopulateUntrackedUploads, :migration, :sidekiq, schema: 20171103140253 do
  include TrackUntrackedUploadsHelpers

  let!(:untracked_files_for_uploads) { table(:untracked_files_for_uploads) }
  let!(:uploads) { table(:uploads) }

  context 'with untracked files and tracked files in untracked_files_for_uploads' do
    let!(:appearance) { create(:appearance, logo: uploaded_file, header_logo: uploaded_file) }
    let!(:user1) { create(:user, :with_avatar) }
    let!(:user2) { create(:user, :with_avatar) }
    let!(:project1) { create(:project, :with_avatar) }
    let!(:project2) { create(:project, :with_avatar) }

    before do
      UploadService.new(project1, uploaded_file, FileUploader).execute # Markdown upload
      UploadService.new(project2, uploaded_file, FileUploader).execute # Markdown upload

      # File records created by PrepareUntrackedUploads
      untracked_files_for_uploads.create!(path: appearance.uploads.first.path)
      untracked_files_for_uploads.create!(path: appearance.uploads.last.path)
      untracked_files_for_uploads.create!(path: user1.uploads.first.path)
      untracked_files_for_uploads.create!(path: user2.uploads.first.path)
      untracked_files_for_uploads.create!(path: project1.uploads.first.path)
      untracked_files_for_uploads.create!(path: project2.uploads.first.path)
      untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project1.full_path}/#{project1.uploads.last.path}")
      untracked_files_for_uploads.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}/#{project2.full_path}/#{project2.uploads.last.path}")

      # Untrack 4 files
      user2.uploads.delete_all
      project2.uploads.delete_all # 2 files: avatar and a Markdown upload
      appearance.uploads.where("path like '%header_logo%'").delete_all
    end

    it 'adds untracked files to the uploads table' do
      expect do
        described_class.new.perform(1, 1000)
      end.to change { uploads.count }.from(4).to(8)

      expect(user2.uploads.count).to eq(1)
      expect(project2.uploads.count).to eq(2)
      expect(appearance.uploads.count).to eq(2)
    end

    it 'sets all added or confirmed tracked files to tracked' do
      expect do
        described_class.new.perform(1, 1000)
      end.to change { untracked_files_for_uploads.where(tracked: true).count }.from(0).to(8)
    end

    it 'does not create duplicate uploads of already tracked files' do
      described_class.new.perform(1, 1000)

      expect(user1.uploads.count).to eq(1)
      expect(project1.uploads.count).to eq(2)
      expect(appearance.uploads.count).to eq(2)
    end

    it 'uses the start and end batch ids [only 1st half]' do
      start_id = untracked_files_for_uploads.all.to_a[0].id
      end_id = untracked_files_for_uploads.all.to_a[3].id

      expect do
        described_class.new.perform(start_id, end_id)
      end.to change { uploads.count }.from(4).to(6)

      expect(user1.uploads.count).to eq(1)
      expect(user2.uploads.count).to eq(1)
      expect(appearance.uploads.count).to eq(2)
      expect(project1.uploads.count).to eq(2)
      expect(project2.uploads.count).to eq(0)

      # Only 4 have been either confirmed or added to uploads
      expect(untracked_files_for_uploads.where(tracked: true).count).to eq(4)
    end

    it 'uses the start and end batch ids [only 2nd half]' do
      start_id = untracked_files_for_uploads.all.to_a[4].id
      end_id = untracked_files_for_uploads.all.to_a[7].id

      expect do
        described_class.new.perform(start_id, end_id)
      end.to change { uploads.count }.from(4).to(6)

      expect(user1.uploads.count).to eq(1)
      expect(user2.uploads.count).to eq(0)
      expect(appearance.uploads.count).to eq(1)
      expect(project1.uploads.count).to eq(2)
      expect(project2.uploads.count).to eq(2)

      # Only 4 have been either confirmed or added to uploads
      expect(untracked_files_for_uploads.where(tracked: true).count).to eq(4)
    end
  end

  context 'with no untracked files' do
    it 'does not add to the uploads table (and does not raise error)' do
      expect do
        described_class.new.perform(1, 1000)
      end.not_to change { uploads.count }.from(0)
    end
  end
end

describe Gitlab::BackgroundMigration::PopulateUntrackedUploads::UntrackedFile do
  include TrackUntrackedUploadsHelpers

  let(:upload_class) { Gitlab::BackgroundMigration::PopulateUntrackedUploads::Upload }

  describe '#ensure_tracked!' do
    let!(:user1) { create(:user, :with_avatar) }
    let!(:untracked_file) { described_class.create!(path: user1.uploads.first.path) }

    context 'when the file is already in the uploads table' do
      it 'does not add an upload' do
        expect do
          untracked_file.ensure_tracked!
        end.not_to change { upload_class.count }.from(1)
      end
    end

    context 'when the file is not already in the uploads table' do
      before do
        user1.uploads.delete_all
      end

      it 'adds an upload' do
        expect do
          untracked_file.ensure_tracked!
        end.to change { upload_class.count }.from(0).to(1)
      end
    end
  end

  describe '#add_to_uploads' do
    shared_examples_for 'add_to_uploads_non_markdown_files' do
      let!(:expected_upload_attrs) { model.uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum') }
      let!(:untracked_file) { described_class.create!(path: expected_upload_attrs['path']) }

      before do
        model.uploads.delete_all
      end

      it 'creates an Upload record' do
        expect do
          untracked_file.add_to_uploads
        end.to change { model.reload.uploads.count }.from(0).to(1)

        expect(model.uploads.first.attributes).to include(expected_upload_attrs)
      end
    end

    context 'for an appearance logo file path' do
      let(:model) { create(:appearance, logo: uploaded_file) }

      it_behaves_like 'add_to_uploads_non_markdown_files'
    end

    context 'for an appearance header_logo file path' do
      let(:model) { create(:appearance, header_logo: uploaded_file) }

      it_behaves_like 'add_to_uploads_non_markdown_files'
    end

    context 'for a pre-Markdown Note attachment file path' do
      class Note < ActiveRecord::Base
        has_many :uploads, as: :model, dependent: :destroy
      end

      let(:model) { create(:note, :with_attachment) }

      it_behaves_like 'add_to_uploads_non_markdown_files'
    end

    context 'for a user avatar file path' do
      let(:model) { create(:user, :with_avatar) }

      it_behaves_like 'add_to_uploads_non_markdown_files'
    end

    context 'for a group avatar file path' do
      let(:model) { create(:group, :with_avatar) }

      it_behaves_like 'add_to_uploads_non_markdown_files'
    end

    context 'for a project avatar file path' do
      let(:model) { create(:project, :with_avatar) }

      it_behaves_like 'add_to_uploads_non_markdown_files'
    end

    context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
      let(:model) { create(:project) }
      let(:expected_upload_attrs) { {} }

      # UntrackedFile.path is different than Upload.path
      let(:untracked_file) { create_untracked_file("/#{model.full_path}/#{model.uploads.first.path}") }

      before do
        # Upload the file
        UploadService.new(model, uploaded_file, FileUploader).execute

        # Create the untracked_files_for_uploads record
        untracked_file

        # Save the expected upload attributes
        expected_upload_attrs = model.reload.uploads.first.attributes.slice('path', 'uploader', 'size', 'checksum')

        # Untrack the file
        model.reload.uploads.delete_all
      end

      it 'creates an Upload record' do
        expect do
          untracked_file.add_to_uploads
        end.to change { model.reload.uploads.count }.from(0).to(1)

        hex_secret = untracked_file.path.match(/\/(\h+)\/rails_sample.jpg/)[1]
        expect(model.uploads.first.attributes).to include(expected_upload_attrs)
      end
    end
  end

  describe '#mark_as_tracked' do
    it 'saves the record with tracked set to true' do
      untracked_file = create_untracked_file("/-/system/appearance/logo/1/some_logo.jpg")

      expect do
        untracked_file.mark_as_tracked
      end.to change { untracked_file.tracked }.from(false).to(true)

      expect(untracked_file.persisted?).to be_truthy
    end
  end

  describe '#upload_path' do
    def assert_upload_path(file_path, expected_upload_path)
      untracked_file = create_untracked_file(file_path)

      expect(untracked_file.upload_path).to eq(expected_upload_path)
    end

    context 'for an appearance logo file path' do
      it 'returns the file path relative to the CarrierWave root' do
        assert_upload_path('/-/system/appearance/logo/1/some_logo.jpg', 'uploads/-/system/appearance/logo/1/some_logo.jpg')
      end
    end

    context 'for an appearance header_logo file path' do
      it 'returns the file path relative to the CarrierWave root' do
        assert_upload_path('/-/system/appearance/header_logo/1/some_logo.jpg', 'uploads/-/system/appearance/header_logo/1/some_logo.jpg')
      end
    end

    context 'for a pre-Markdown Note attachment file path' do
      it 'returns the file path relative to the CarrierWave root' do
        assert_upload_path('/-/system/note/attachment/1234/some_attachment.pdf', 'uploads/-/system/note/attachment/1234/some_attachment.pdf')
      end
    end

    context 'for a user avatar file path' do
      it 'returns the file path relative to the CarrierWave root' do
        assert_upload_path('/-/system/user/avatar/1234/avatar.jpg', 'uploads/-/system/user/avatar/1234/avatar.jpg')
      end
    end

    context 'for a group avatar file path' do
      it 'returns the file path relative to the CarrierWave root' do
        assert_upload_path('/-/system/group/avatar/1234/avatar.jpg', 'uploads/-/system/group/avatar/1234/avatar.jpg')
      end
    end

    context 'for a project avatar file path' do
      it 'returns the file path relative to the CarrierWave root' do
        assert_upload_path('/-/system/project/avatar/1234/avatar.jpg', 'uploads/-/system/project/avatar/1234/avatar.jpg')
      end
    end

    context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
      it 'returns the file path relative to the project directory in uploads' do
        project = create(:project)
        random_hex = SecureRandom.hex

        assert_upload_path("/#{project.full_path}/#{random_hex}/Some file.jpg", "#{random_hex}/Some file.jpg")
      end
    end
  end

  describe '#uploader' do
    def assert_uploader(file_path, expected_uploader)
      untracked_file = create_untracked_file(file_path)

      expect(untracked_file.uploader).to eq(expected_uploader)
    end

    context 'for an appearance logo file path' do
      it 'returns AttachmentUploader as a string' do
        assert_uploader('/-/system/appearance/logo/1/some_logo.jpg', 'AttachmentUploader')
      end
    end

    context 'for an appearance header_logo file path' do
      it 'returns AttachmentUploader as a string' do
        assert_uploader('/-/system/appearance/header_logo/1/some_logo.jpg', 'AttachmentUploader')
      end
    end

    context 'for a pre-Markdown Note attachment file path' do
      it 'returns AttachmentUploader as a string' do
        assert_uploader('/-/system/note/attachment/1234/some_attachment.pdf', 'AttachmentUploader')
      end
    end

    context 'for a user avatar file path' do
      it 'returns AvatarUploader as a string' do
        assert_uploader('/-/system/user/avatar/1234/avatar.jpg', 'AvatarUploader')
      end
    end

    context 'for a group avatar file path' do
      it 'returns AvatarUploader as a string' do
        assert_uploader('/-/system/group/avatar/1234/avatar.jpg', 'AvatarUploader')
      end
    end

    context 'for a project avatar file path' do
      it 'returns AvatarUploader as a string' do
        assert_uploader('/-/system/project/avatar/1234/avatar.jpg', 'AvatarUploader')
      end
    end

    context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
      it 'returns FileUploader as a string' do
        project = create(:project)

        assert_uploader("/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg", 'FileUploader')
      end
    end
  end

  describe '#model_type' do
    def assert_model_type(file_path, expected_model_type)
      untracked_file = create_untracked_file(file_path)

      expect(untracked_file.model_type).to eq(expected_model_type)
    end

    context 'for an appearance logo file path' do
      it 'returns Appearance as a string' do
        assert_model_type('/-/system/appearance/logo/1/some_logo.jpg', 'Appearance')
      end
    end

    context 'for an appearance header_logo file path' do
      it 'returns Appearance as a string' do
        assert_model_type('/-/system/appearance/header_logo/1/some_logo.jpg', 'Appearance')
      end
    end

    context 'for a pre-Markdown Note attachment file path' do
      it 'returns Note as a string' do
        assert_model_type('/-/system/note/attachment/1234/some_attachment.pdf', 'Note')
      end
    end

    context 'for a user avatar file path' do
      it 'returns User as a string' do
        assert_model_type('/-/system/user/avatar/1234/avatar.jpg', 'User')
      end
    end

    context 'for a group avatar file path' do
      it 'returns Namespace as a string' do
        assert_model_type('/-/system/group/avatar/1234/avatar.jpg', 'Namespace')
      end
    end

    context 'for a project avatar file path' do
      it 'returns Project as a string' do
        assert_model_type('/-/system/project/avatar/1234/avatar.jpg', 'Project')
      end
    end

    context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
      it 'returns Project as a string' do
        project = create(:project)

        assert_model_type("/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg", 'Project')
      end
    end
  end

  describe '#model_id' do
    def assert_model_id(file_path, expected_model_id)
      untracked_file = create_untracked_file(file_path)

      expect(untracked_file.model_id).to eq(expected_model_id)
    end

    context 'for an appearance logo file path' do
      it 'returns the ID as a string' do
        assert_model_id('/-/system/appearance/logo/1/some_logo.jpg', '1')
      end
    end

    context 'for an appearance header_logo file path' do
      it 'returns the ID as a string' do
        assert_model_id('/-/system/appearance/header_logo/1/some_logo.jpg', '1')
      end
    end

    context 'for a pre-Markdown Note attachment file path' do
      it 'returns the ID as a string' do
        assert_model_id('/-/system/note/attachment/1234/some_attachment.pdf', '1234')
      end
    end

    context 'for a user avatar file path' do
      it 'returns the ID as a string' do
        assert_model_id('/-/system/user/avatar/1234/avatar.jpg', '1234')
      end
    end

    context 'for a group avatar file path' do
      it 'returns the ID as a string' do
        assert_model_id('/-/system/group/avatar/1234/avatar.jpg', '1234')
      end
    end

    context 'for a project avatar file path' do
      it 'returns the ID as a string' do
        assert_model_id('/-/system/project/avatar/1234/avatar.jpg', '1234')
      end
    end

    context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
      it 'returns the ID as a string' do
        project = create(:project)

        assert_model_id("/#{project.full_path}/#{SecureRandom.hex}/Some file.jpg", project.id.to_s)
      end
    end
  end

  describe '#file_size' do
    context 'for an appearance logo file path' do
      let(:appearance) { create(:appearance, logo: uploaded_file) }
      let(:untracked_file) { described_class.create!(path: appearance.uploads.first.path) }

      it 'returns the file size' do
        expect(untracked_file.file_size).to eq(35255)
      end

      it 'returns the same thing that CarrierWave would return' do
        expect(untracked_file.file_size).to eq(appearance.logo.size)
      end
    end

    context 'for a project avatar file path' do
      let(:project) { create(:project, avatar: uploaded_file) }
      let(:untracked_file) { described_class.create!(path: project.uploads.first.path) }

      it 'returns the file size' do
        expect(untracked_file.file_size).to eq(35255)
      end

      it 'returns the same thing that CarrierWave would return' do
        expect(untracked_file.file_size).to eq(project.avatar.size)
      end
    end

    context 'for a project Markdown attachment (notes, issues, MR descriptions) file path' do
      let(:project) { create(:project) }
      let(:untracked_file) { create_untracked_file("/#{project.full_path}/#{project.uploads.first.path}") }

      before do
        UploadService.new(project, uploaded_file, FileUploader).execute
      end

      it 'returns the file size' do
        expect(untracked_file.file_size).to eq(35255)
      end

      it 'returns the same thing that CarrierWave would return' do
        expect(untracked_file.file_size).to eq(project.uploads.first.size)
      end
    end
  end

  def create_untracked_file(path_relative_to_upload_dir)
    described_class.create!(path: "#{Gitlab::BackgroundMigration::PrepareUntrackedUploads::RELATIVE_UPLOAD_DIR}#{path_relative_to_upload_dir}")
  end
end