diff --git a/app/helpers/storage_helper.rb b/app/helpers/storage_helper.rb index e80b3f2b54a..ecf37bae6b3 100644 --- a/app/helpers/storage_helper.rb +++ b/app/helpers/storage_helper.rb @@ -12,10 +12,11 @@ module StorageHelper def storage_counters_details(statistics) counters = { counter_repositories: storage_counter(statistics.repository_size), + counter_wikis: storage_counter(statistics.wiki_size), counter_build_artifacts: storage_counter(statistics.build_artifacts_size), counter_lfs_objects: storage_counter(statistics.lfs_objects_size) } - _("%{counter_repositories} repositories, %{counter_build_artifacts} build artifacts, %{counter_lfs_objects} LFS") % counters + _("%{counter_repositories} repositories, %{counter_wikis} wikis, %{counter_build_artifacts} build artifacts, %{counter_lfs_objects} LFS") % counters end end diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 7393ef4b05c..f7c31890198 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -76,6 +76,7 @@ class Namespace < ApplicationRecord 'namespaces.*', 'COALESCE(SUM(ps.storage_size), 0) AS storage_size', 'COALESCE(SUM(ps.repository_size), 0) AS repository_size', + 'COALESCE(SUM(ps.wiki_size), 0) AS wiki_size', 'COALESCE(SUM(ps.lfs_objects_size), 0) AS lfs_objects_size', 'COALESCE(SUM(ps.build_artifacts_size), 0) AS build_artifacts_size', 'COALESCE(SUM(ps.packages_size), 0) AS packages_size' diff --git a/app/models/project_statistics.rb b/app/models/project_statistics.rb index 6fe8cb40d25..832c8417b5b 100644 --- a/app/models/project_statistics.rb +++ b/app/models/project_statistics.rb @@ -4,9 +4,16 @@ class ProjectStatistics < ApplicationRecord belongs_to :project belongs_to :namespace + default_value_for :wiki_size, 0 + + # older migrations fail due to non-existent attribute without this + def wiki_size + has_attribute?(:wiki_size) ? super : 0 + end + before_save :update_storage_size - COLUMNS_TO_REFRESH = [:repository_size, :lfs_objects_size, :commit_count].freeze + COLUMNS_TO_REFRESH = [:repository_size, :wiki_size, :lfs_objects_size, :commit_count].freeze INCREMENTABLE_COLUMNS = { build_artifacts_size: %i[storage_size], packages_size: %i[storage_size] }.freeze def total_repository_size @@ -27,11 +34,14 @@ class ProjectStatistics < ApplicationRecord self.commit_count = project.repository.commit_count end - # Repository#size needs to be converted from MB to Byte. def update_repository_size self.repository_size = project.repository.size * 1.megabyte end + def update_wiki_size + self.wiki_size = project.wiki.repository.size * 1.megabyte + end + def update_lfs_objects_size self.lfs_objects_size = project.lfs_objects.sum(:size) end @@ -42,7 +52,7 @@ class ProjectStatistics < ApplicationRecord end def update_storage_size - self.storage_size = repository_size + lfs_objects_size + build_artifacts_size + packages_size + self.storage_size = repository_size + wiki_size + lfs_objects_size + build_artifacts_size + packages_size end # Since this incremental update method does not call update_storage_size above, diff --git a/app/services/projects/update_statistics_service.rb b/app/services/projects/update_statistics_service.rb index f32a779fab0..28677a398f3 100644 --- a/app/services/projects/update_statistics_service.rb +++ b/app/services/projects/update_statistics_service.rb @@ -3,7 +3,7 @@ module Projects class UpdateStatisticsService < BaseService def execute - return unless project && project.repository.exists? + return unless project Rails.logger.info("Updating statistics for project #{project.id}") diff --git a/app/workers/post_receive.rb b/app/workers/post_receive.rb index 9a9c0c9d803..3f1639ec2ed 100644 --- a/app/workers/post_receive.rb +++ b/app/workers/post_receive.rb @@ -74,6 +74,8 @@ class PostReceive def process_wiki_changes(post_received) post_received.project.touch(:last_activity_at, :last_repository_updated_at) + post_received.project.wiki.repository.expire_statistics_caches + ProjectCacheWorker.perform_async(post_received.project.id, [], [:wiki_size]) end def log(message) diff --git a/app/workers/project_cache_worker.rb b/app/workers/project_cache_worker.rb index b2e0701008a..4e8ea903139 100644 --- a/app/workers/project_cache_worker.rb +++ b/app/workers/project_cache_worker.rb @@ -16,10 +16,12 @@ class ProjectCacheWorker def perform(project_id, files = [], statistics = []) project = Project.find_by(id: project_id) - return unless project && project.repository.exists? + return unless project update_statistics(project, statistics) + return unless project.repository.exists? + project.repository.refresh_method_caches(files.map(&:to_sym)) project.cleanup diff --git a/changelogs/unreleased/add-wiki-size-to-statistics.yml b/changelogs/unreleased/add-wiki-size-to-statistics.yml new file mode 100644 index 00000000000..85b6d7a1774 --- /dev/null +++ b/changelogs/unreleased/add-wiki-size-to-statistics.yml @@ -0,0 +1,5 @@ +--- +title: Add wiki size to project statistics +merge_request: 25321 +author: Peter Marko +type: added diff --git a/db/migrate/20190527194830_add_wiki_size_to_statistics.rb b/db/migrate/20190527194830_add_wiki_size_to_statistics.rb new file mode 100644 index 00000000000..d4f16cdec18 --- /dev/null +++ b/db/migrate/20190527194830_add_wiki_size_to_statistics.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class AddWikiSizeToStatistics < ActiveRecord::Migration[5.0] + DOWNTIME = false + + def change + add_column :project_statistics, :wiki_size, :bigint + end +end diff --git a/db/post_migrate/20190527194900_schedule_calculate_wiki_sizes.rb b/db/post_migrate/20190527194900_schedule_calculate_wiki_sizes.rb new file mode 100644 index 00000000000..04cf5906b61 --- /dev/null +++ b/db/post_migrate/20190527194900_schedule_calculate_wiki_sizes.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +class ScheduleCalculateWikiSizes < ActiveRecord::Migration[5.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + MIGRATION = 'CalculateWikiSizes' + BATCH_SIZE = 100000 + BATCH_TIME = 5.minutes + + class ProjectStatistics < ActiveRecord::Base + self.table_name = 'project_statistics' + + scope :without_wiki_size, -> { where(wiki_size: nil) } + + include ::EachBatch + end + + disable_ddl_transaction! + + def up + queue_background_migration_jobs_by_range_at_intervals( + ::ScheduleCalculateWikiSizes::ProjectStatistics.without_wiki_size, + MIGRATION, + BATCH_TIME, + batch_size: BATCH_SIZE) + end + + def down + # no-op + end +end diff --git a/db/schema.rb b/db/schema.rb index bb59af540fe..89140048ad3 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20190524062810) do +ActiveRecord::Schema.define(version: 20190527194900) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -1745,6 +1745,7 @@ ActiveRecord::Schema.define(version: 20190524062810) do t.bigint "lfs_objects_size", default: 0, null: false t.bigint "build_artifacts_size", default: 0, null: false t.bigint "packages_size" + t.bigint "wiki_size" t.index ["namespace_id"], name: "index_project_statistics_on_namespace_id", using: :btree t.index ["project_id"], name: "index_project_statistics_on_project_id", unique: true, using: :btree end diff --git a/doc/api/groups.md b/doc/api/groups.md index 907b443d355..20789a1d4a4 100644 --- a/doc/api/groups.md +++ b/doc/api/groups.md @@ -68,6 +68,7 @@ GET /groups?statistics=true "statistics": { "storage_size" : 212, "repository_size" : 33, + "wiki_size" : 100, "lfs_objects_size" : 123, "job_artifacts_size" : 57 diff --git a/doc/api/projects.md b/doc/api/projects.md index 951961e45ff..75669d85803 100644 --- a/doc/api/projects.md +++ b/doc/api/projects.md @@ -153,6 +153,7 @@ When the user is authenticated and `simple` is not set this returns something li "commit_count": 37, "storage_size": 1038090, "repository_size": 1038090, + "wiki_size" : 0, "lfs_objects_size": 0, "job_artifacts_size": 0 }, @@ -234,6 +235,7 @@ When the user is authenticated and `simple` is not set this returns something li "commit_count": 12, "storage_size": 2066080, "repository_size": 2066080, + "wiki_size" : 0, "lfs_objects_size": 0, "job_artifacts_size": 0 }, @@ -342,6 +344,7 @@ GET /users/:user_id/projects "commit_count": 37, "storage_size": 1038090, "repository_size": 1038090, + "wiki_size" : 0, "lfs_objects_size": 0, "job_artifacts_size": 0 }, @@ -423,6 +426,7 @@ GET /users/:user_id/projects "commit_count": 12, "storage_size": 2066080, "repository_size": 2066080, + "wiki_size" : 0, "lfs_objects_size": 0, "job_artifacts_size": 0 }, @@ -548,6 +552,7 @@ GET /projects/:id "commit_count": 37, "storage_size": 1038090, "repository_size": 1038090, + "wiki_size" : 0, "lfs_objects_size": 0, "job_artifacts_size": 0 }, diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 3d7cf2c0bb1..4e99fec02dd 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -303,6 +303,7 @@ module API expose :commit_count expose :storage_size expose :repository_size + expose :wiki_size expose :lfs_objects_size expose :build_artifacts_size, as: :job_artifacts_size end @@ -355,6 +356,7 @@ module API with_options format_with: -> (value) { value.to_i } do expose :storage_size expose :repository_size + expose :wiki_size expose :lfs_objects_size expose :build_artifacts_size, as: :job_artifacts_size end diff --git a/lib/gitlab/background_migration/calculate_wiki_sizes.rb b/lib/gitlab/background_migration/calculate_wiki_sizes.rb new file mode 100644 index 00000000000..886c41a2b9d --- /dev/null +++ b/lib/gitlab/background_migration/calculate_wiki_sizes.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class CalculateWikiSizes + def perform(start_id, stop_id) + ::ProjectStatistics.where(wiki_size: nil) + .where(id: start_id..stop_id) + .includes(project: [:route, :group, namespace: [:owner]]).find_each do |statistics| + statistics.refresh!(only: [:wiki_size]) + rescue => e + Rails.logger.error "Failed to update wiki statistics. id: #{statistics.id} message: #{e.message}" + end + end + end + end +end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index af53da84bea..81a0ad46b77 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -107,7 +107,7 @@ msgstr "" msgid "%{commit_author_link} authored %{commit_timeago}" msgstr "" -msgid "%{counter_repositories} repositories, %{counter_build_artifacts} build artifacts, %{counter_lfs_objects} LFS" +msgid "%{counter_repositories} repositories, %{counter_wikis} wikis, %{counter_build_artifacts} build artifacts, %{counter_lfs_objects} LFS" msgstr "" msgid "%{count} more" diff --git a/spec/features/admin/admin_sees_project_statistics_spec.rb b/spec/features/admin/admin_sees_project_statistics_spec.rb index 95d1fc5b57a..b5323a1c76d 100644 --- a/spec/features/admin/admin_sees_project_statistics_spec.rb +++ b/spec/features/admin/admin_sees_project_statistics_spec.rb @@ -15,7 +15,7 @@ describe "Admin > Admin sees project statistics" do let(:project) { create(:project, :repository) } it "shows project statistics" do - expect(page).to have_content("Storage: 0 Bytes (0 Bytes repositories, 0 Bytes build artifacts, 0 Bytes LFS)") + expect(page).to have_content("Storage: 0 Bytes (0 Bytes repositories, 0 Bytes wikis, 0 Bytes build artifacts, 0 Bytes LFS)") end end diff --git a/spec/helpers/storage_helper_spec.rb b/spec/helpers/storage_helper_spec.rb index 50c74a7c2f9..62c00964524 100644 --- a/spec/helpers/storage_helper_spec.rb +++ b/spec/helpers/storage_helper_spec.rb @@ -26,11 +26,12 @@ describe StorageHelper do namespace: namespace, statistics: build(:project_statistics, repository_size: 10.kilobytes, + wiki_size: 10.bytes, lfs_objects_size: 20.gigabytes, build_artifacts_size: 30.megabytes)) end - let(:message) { '10 KB repositories, 30 MB build artifacts, 20 GB LFS' } + let(:message) { '10 KB repositories, 10 Bytes wikis, 30 MB build artifacts, 20 GB LFS' } it 'works on ProjectStatistics' do expect(helper.storage_counters_details(project.statistics)).to eq(message) diff --git a/spec/lib/gitlab/background_migration/schedule_calculate_wiki_sizes_spec.rb b/spec/lib/gitlab/background_migration/schedule_calculate_wiki_sizes_spec.rb new file mode 100644 index 00000000000..d494ce68c5b --- /dev/null +++ b/spec/lib/gitlab/background_migration/schedule_calculate_wiki_sizes_spec.rb @@ -0,0 +1,62 @@ +require 'spec_helper' +require Rails.root.join('db', 'post_migrate', '20190527194900_schedule_calculate_wiki_sizes.rb') + +describe ScheduleCalculateWikiSizes, :migration, :sidekiq do + let(:migration_class) { Gitlab::BackgroundMigration::CalculateWikiSizes } + let(:migration_name) { migration_class.to_s.demodulize } + + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:project_statistics) { table(:project_statistics) } + + context 'when missing wiki sizes exist' do + before do + namespaces.create!(id: 1, name: 'wiki-migration', path: 'wiki-migration') + projects.create!(id: 1, name: 'wiki-project-1', path: 'wiki-project-1', namespace_id: 1) + projects.create!(id: 2, name: 'wiki-project-2', path: 'wiki-project-2', namespace_id: 1) + projects.create!(id: 3, name: 'wiki-project-3', path: 'wiki-project-3', namespace_id: 1) + project_statistics.create!(id: 1, project_id: 1, namespace_id: 1, wiki_size: 1000) + project_statistics.create!(id: 2, project_id: 2, namespace_id: 1, wiki_size: nil) + project_statistics.create!(id: 3, project_id: 3, namespace_id: 1, wiki_size: nil) + end + + it 'schedules a background migration' do + Sidekiq::Testing.fake! do + Timecop.freeze do + migrate! + + expect(migration_name).to be_scheduled_delayed_migration(5.minutes, 2, 3) + expect(BackgroundMigrationWorker.jobs.size).to eq 1 + end + end + end + + it 'calculates missing wiki sizes' do + expect(project_statistics.find_by(id: 2).wiki_size).to be_nil + expect(project_statistics.find_by(id: 3).wiki_size).to be_nil + + migrate! + + expect(project_statistics.find_by(id: 2).wiki_size).not_to be_nil + expect(project_statistics.find_by(id: 3).wiki_size).not_to be_nil + end + end + + context 'when missing wiki sizes do not exist' do + before do + namespaces.create!(id: 1, name: 'wiki-migration', path: 'wiki-migration') + projects.create!(id: 1, name: 'wiki-project-1', path: 'wiki-project-1', namespace_id: 1) + project_statistics.create!(id: 1, project_id: 1, namespace_id: 1, wiki_size: 1000) + end + + it 'does not schedule a background migration' do + Sidekiq::Testing.fake! do + Timecop.freeze do + migrate! + + expect(BackgroundMigrationWorker.jobs.size).to eq 0 + end + end + end + end +end diff --git a/spec/models/namespace_spec.rb b/spec/models/namespace_spec.rb index bfde367c47f..d80183af33e 100644 --- a/spec/models/namespace_spec.rb +++ b/spec/models/namespace_spec.rb @@ -147,6 +147,7 @@ describe Namespace do namespace: namespace, statistics: build(:project_statistics, repository_size: 101, + wiki_size: 505, lfs_objects_size: 202, build_artifacts_size: 303, packages_size: 404)) @@ -157,6 +158,7 @@ describe Namespace do namespace: namespace, statistics: build(:project_statistics, repository_size: 10, + wiki_size: 50, lfs_objects_size: 20, build_artifacts_size: 30, packages_size: 40)) @@ -167,8 +169,9 @@ describe Namespace do project2 statistics = described_class.with_statistics.find(namespace.id) - expect(statistics.storage_size).to eq 1110 + expect(statistics.storage_size).to eq 1665 expect(statistics.repository_size).to eq 111 + expect(statistics.wiki_size).to eq 555 expect(statistics.lfs_objects_size).to eq 222 expect(statistics.build_artifacts_size).to eq 333 expect(statistics.packages_size).to eq 444 @@ -179,6 +182,7 @@ describe Namespace do expect(statistics.storage_size).to eq 0 expect(statistics.repository_size).to eq 0 + expect(statistics.wiki_size).to eq 0 expect(statistics.lfs_objects_size).to eq 0 expect(statistics.build_artifacts_size).to eq 0 expect(statistics.packages_size).to eq 0 diff --git a/spec/models/project_statistics_spec.rb b/spec/models/project_statistics_spec.rb index 738398a06f9..f985c114d4b 100644 --- a/spec/models/project_statistics_spec.rb +++ b/spec/models/project_statistics_spec.rb @@ -16,16 +16,18 @@ describe ProjectStatistics do statistics.update!( commit_count: 8.exabytes - 1, repository_size: 2.exabytes, + wiki_size: 1.exabytes, lfs_objects_size: 2.exabytes, - build_artifacts_size: 4.exabytes - 1 + build_artifacts_size: 3.exabytes - 1 ) statistics.reload expect(statistics.commit_count).to eq(8.exabytes - 1) expect(statistics.repository_size).to eq(2.exabytes) + expect(statistics.wiki_size).to eq(1.exabytes) expect(statistics.lfs_objects_size).to eq(2.exabytes) - expect(statistics.build_artifacts_size).to eq(4.exabytes - 1) + expect(statistics.build_artifacts_size).to eq(3.exabytes - 1) expect(statistics.storage_size).to eq(8.exabytes - 1) end end @@ -33,6 +35,7 @@ describe ProjectStatistics do describe '#total_repository_size' do it "sums repository and LFS object size" do statistics.repository_size = 2 + statistics.wiki_size = 6 statistics.lfs_objects_size = 3 statistics.build_artifacts_size = 4 @@ -40,10 +43,17 @@ describe ProjectStatistics do end end + describe '#wiki_size' do + it "is initialized with not null value" do + expect(statistics.wiki_size).to eq 0 + end + end + describe '#refresh!' do before do allow(statistics).to receive(:update_commit_count) allow(statistics).to receive(:update_repository_size) + allow(statistics).to receive(:update_wiki_size) allow(statistics).to receive(:update_lfs_objects_size) allow(statistics).to receive(:update_storage_size) end @@ -56,6 +66,7 @@ describe ProjectStatistics do it "sums all counters" do expect(statistics).to have_received(:update_commit_count) expect(statistics).to have_received(:update_repository_size) + expect(statistics).to have_received(:update_wiki_size) expect(statistics).to have_received(:update_lfs_objects_size) end end @@ -69,6 +80,45 @@ describe ProjectStatistics do expect(statistics).to have_received(:update_lfs_objects_size) expect(statistics).not_to have_received(:update_commit_count) expect(statistics).not_to have_received(:update_repository_size) + expect(statistics).not_to have_received(:update_wiki_size) + end + end + + context 'without repositories' do + it 'does not crash' do + expect(project.repository.exists?).to be_falsey + expect(project.wiki.repository.exists?).to be_falsey + + statistics.refresh! + + expect(statistics).to have_received(:update_commit_count) + expect(statistics).to have_received(:update_repository_size) + expect(statistics).to have_received(:update_wiki_size) + expect(statistics.repository_size).to eq(0) + expect(statistics.commit_count).to eq(0) + expect(statistics.wiki_size).to eq(0) + end + end + + context 'with deleted repositories' do + let(:project) { create(:project, :repository, :wiki_repo) } + + before do + Gitlab::GitalyClient::StorageSettings.allow_disk_access do + FileUtils.rm_rf(project.repository.path) + FileUtils.rm_rf(project.wiki.repository.path) + end + end + + it 'does not crash' do + statistics.refresh! + + expect(statistics).to have_received(:update_commit_count) + expect(statistics).to have_received(:update_repository_size) + expect(statistics).to have_received(:update_wiki_size) + expect(statistics.repository_size).to eq(0) + expect(statistics.commit_count).to eq(0) + expect(statistics.wiki_size).to eq(0) end end end @@ -95,6 +145,17 @@ describe ProjectStatistics do end end + describe '#update_wiki_size' do + before do + allow(project.wiki.repository).to receive(:size).and_return(34) + statistics.update_wiki_size + end + + it "stores the size of the wiki" do + expect(statistics.wiki_size).to eq 34.megabytes + end + end + describe '#update_lfs_objects_size' do let!(:lfs_object1) { create(:lfs_object, size: 23.megabytes) } let!(:lfs_object2) { create(:lfs_object, size: 34.megabytes) } @@ -114,12 +175,13 @@ describe ProjectStatistics do it "sums all storage counters" do statistics.update!( repository_size: 2, + wiki_size: 4, lfs_objects_size: 3 ) statistics.reload - expect(statistics.storage_size).to eq 5 + expect(statistics.storage_size).to eq 9 end end diff --git a/spec/requests/api/groups_spec.rb b/spec/requests/api/groups_spec.rb index 7176bc23e34..c41408fba65 100644 --- a/spec/requests/api/groups_spec.rb +++ b/spec/requests/api/groups_spec.rb @@ -90,8 +90,9 @@ describe API::Groups do it "includes statistics if requested" do attributes = { - storage_size: 702, + storage_size: 1158, repository_size: 123, + wiki_size: 456, lfs_objects_size: 234, build_artifacts_size: 345 }.stringify_keys diff --git a/spec/services/projects/update_statistics_service_spec.rb b/spec/services/projects/update_statistics_service_spec.rb index 5000ea58e5f..8534853fbc7 100644 --- a/spec/services/projects/update_statistics_service_spec.rb +++ b/spec/services/projects/update_statistics_service_spec.rb @@ -17,19 +17,9 @@ describe Projects::UpdateStatisticsService do end end - context 'with an existing project without a repository' do + context 'with an existing project' do let(:project) { create(:project) } - it 'does nothing' do - expect_any_instance_of(ProjectStatistics).not_to receive(:refresh!) - - service.execute - end - end - - context 'with an existing project with a repository' do - let(:project) { create(:project, :repository) } - it 'refreshes the project statistics' do expect_any_instance_of(ProjectStatistics).to receive(:refresh!) .with(only: statistics.map(&:to_sym)) diff --git a/spec/workers/project_cache_worker_spec.rb b/spec/workers/project_cache_worker_spec.rb index 3c40269adc7..51afb076da1 100644 --- a/spec/workers/project_cache_worker_spec.rb +++ b/spec/workers/project_cache_worker_spec.rb @@ -25,10 +25,11 @@ describe ProjectCacheWorker do end context 'with an existing project without a repository' do - it 'does nothing' do + it 'updates statistics but does not refresh the method cashes' do allow_any_instance_of(Repository).to receive(:exists?).and_return(false) - expect(worker).not_to receive(:update_statistics) + expect(worker).to receive(:update_statistics) + expect_any_instance_of(Repository).not_to receive(:refresh_method_caches) worker.perform(project.id) end