kotovalexarian-likes-gitlab/gitlab-org--gitlab-foss
1
0
Fork 0
Code Releases Activity

Merge branch 'add-wiki-size-to-statistics' into 'master'

Browse source 
Add wiki size to project statistics

Closes #57822

See merge request gitlab-org/gitlab-ce!25321
This commit is contained in:
Stan Hu 2019-05-29 22:01:50 +00:00
commit 82f39315b1
23 changed files with 238 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
app/helpers/storage_helper.rb
View file

@ -12,10 +12,11 @@ module StorageHelper
def storage_counters_details(statistics)
counters = {
counter_repositories: storage_counter(statistics.repository_size),
counter_wikis: storage_counter(statistics.wiki_size),
counter_build_artifacts: storage_counter(statistics.build_artifacts_size),
counter_lfs_objects: storage_counter(statistics.lfs_objects_size)
}
_("%{counter_repositories} repositories, %{counter_build_artifacts} build artifacts, %{counter_lfs_objects} LFS") % counters
_("%{counter_repositories} repositories, %{counter_wikis} wikis, %{counter_build_artifacts} build artifacts, %{counter_lfs_objects} LFS") % counters
end
end

1
app/models/namespace.rb
View file

@ -76,6 +76,7 @@ class Namespace < ApplicationRecord
'namespaces.*',
'COALESCE(SUM(ps.storage_size), 0) AS storage_size',
'COALESCE(SUM(ps.repository_size), 0) AS repository_size',
'COALESCE(SUM(ps.wiki_size), 0) AS wiki_size',
'COALESCE(SUM(ps.lfs_objects_size), 0) AS lfs_objects_size',
'COALESCE(SUM(ps.build_artifacts_size), 0) AS build_artifacts_size',
'COALESCE(SUM(ps.packages_size), 0) AS packages_size'

16
app/models/project_statistics.rb
View file

@ -4,9 +4,16 @@ class ProjectStatistics < ApplicationRecord
belongs_to :project
belongs_to :namespace
default_value_for :wiki_size, 0
# older migrations fail due to non-existent attribute without this
def wiki_size
has_attribute?(:wiki_size) ? super : 0
end
before_save :update_storage_size
COLUMNS_TO_REFRESH = [:repository_size, :lfs_objects_size, :commit_count].freeze
COLUMNS_TO_REFRESH = [:repository_size, :wiki_size, :lfs_objects_size, :commit_count].freeze
INCREMENTABLE_COLUMNS = { build_artifacts_size: %i[storage_size], packages_size: %i[storage_size] }.freeze
def total_repository_size
@ -27,11 +34,14 @@ class ProjectStatistics < ApplicationRecord
self.commit_count = project.repository.commit_count
end
# Repository#size needs to be converted from MB to Byte.
def update_repository_size
self.repository_size = project.repository.size * 1.megabyte
end
def update_wiki_size
self.wiki_size = project.wiki.repository.size * 1.megabyte
end
def update_lfs_objects_size
self.lfs_objects_size = project.lfs_objects.sum(:size)
end
@ -42,7 +52,7 @@ class ProjectStatistics < ApplicationRecord
end
def update_storage_size
self.storage_size = repository_size + lfs_objects_size + build_artifacts_size + packages_size
self.storage_size = repository_size + wiki_size + lfs_objects_size + build_artifacts_size + packages_size
end
# Since this incremental update method does not call update_storage_size above,

2
app/services/projects/update_statistics_service.rb
View file

@ -3,7 +3,7 @@
module Projects
class UpdateStatisticsService < BaseService
def execute
return unless project && project.repository.exists?
return unless project
Rails.logger.info("Updating statistics for project #{project.id}")

2
app/workers/post_receive.rb
View file

@ -74,6 +74,8 @@ class PostReceive
def process_wiki_changes(post_received)
post_received.project.touch(:last_activity_at, :last_repository_updated_at)
post_received.project.wiki.repository.expire_statistics_caches
ProjectCacheWorker.perform_async(post_received.project.id, [], [:wiki_size])
end
def log(message)

4
app/workers/project_cache_worker.rb
View file

@ -16,10 +16,12 @@ class ProjectCacheWorker
def perform(project_id, files = [], statistics = [])
project = Project.find_by(id: project_id)
return unless project && project.repository.exists?
return unless project
update_statistics(project, statistics)
return unless project.repository.exists?
project.repository.refresh_method_caches(files.map(&:to_sym))
project.cleanup

5
changelogs/unreleased/add-wiki-size-to-statistics.yml Normal file
View file

@ -0,0 +1,5 @@
---
title: Add wiki size to project statistics
merge_request: 25321
author: Peter Marko
type: added

9
db/migrate/20190527194830_add_wiki_size_to_statistics.rb Normal file
View file

@ -0,0 +1,9 @@
# frozen_string_literal: true
class AddWikiSizeToStatistics < ActiveRecord::Migration[5.0]
DOWNTIME = false
def change
add_column :project_statistics, :wiki_size, :bigint
end
end

32
db/post_migrate/20190527194900_schedule_calculate_wiki_sizes.rb Normal file
View file

@ -0,0 +1,32 @@
# frozen_string_literal: true
class ScheduleCalculateWikiSizes < ActiveRecord::Migration[5.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
MIGRATION = 'CalculateWikiSizes'
BATCH_SIZE = 100000
BATCH_TIME = 5.minutes
class ProjectStatistics < ActiveRecord::Base
self.table_name = 'project_statistics'
scope :without_wiki_size, -> { where(wiki_size: nil) }
include ::EachBatch
end
disable_ddl_transaction!
def up
queue_background_migration_jobs_by_range_at_intervals(
::ScheduleCalculateWikiSizes::ProjectStatistics.without_wiki_size,
MIGRATION,
BATCH_TIME,
batch_size: BATCH_SIZE)
end
def down
# no-op
end
end

3
db/schema.rb
View file

@ -10,7 +10,7 @@
#
# It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 20190524062810) do
ActiveRecord::Schema.define(version: 20190527194900) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
@ -1745,6 +1745,7 @@ ActiveRecord::Schema.define(version: 20190524062810) do
t.bigint "lfs_objects_size", default: 0, null: false
t.bigint "build_artifacts_size", default: 0, null: false
t.bigint "packages_size"
t.bigint "wiki_size"
t.index ["namespace_id"], name: "index_project_statistics_on_namespace_id", using: :btree
t.index ["project_id"], name: "index_project_statistics_on_project_id", unique: true, using: :btree
end

1
doc/api/groups.md
View file

@ -68,6 +68,7 @@ GET /groups?statistics=true
"statistics": {
"storage_size" : 212,
"repository_size" : 33,
"wiki_size" : 100,
"lfs_objects_size" : 123,
"job_artifacts_size" : 57

5
doc/api/projects.md
View file

@ -153,6 +153,7 @@ When the user is authenticated and `simple` is not set this returns something li
"commit_count": 37,
"storage_size": 1038090,
"repository_size": 1038090,
"wiki_size" : 0,
"lfs_objects_size": 0,
"job_artifacts_size": 0
},
@ -234,6 +235,7 @@ When the user is authenticated and `simple` is not set this returns something li
"commit_count": 12,
"storage_size": 2066080,
"repository_size": 2066080,
"wiki_size" : 0,
"lfs_objects_size": 0,
"job_artifacts_size": 0
},
@ -342,6 +344,7 @@ GET /users/:user_id/projects
"commit_count": 37,
"storage_size": 1038090,
"repository_size": 1038090,
"wiki_size" : 0,
"lfs_objects_size": 0,
"job_artifacts_size": 0
},
@ -423,6 +426,7 @@ GET /users/:user_id/projects
"commit_count": 12,
"storage_size": 2066080,
"repository_size": 2066080,
"wiki_size" : 0,
"lfs_objects_size": 0,
"job_artifacts_size": 0
},
@ -548,6 +552,7 @@ GET /projects/:id
"commit_count": 37,
"storage_size": 1038090,
"repository_size": 1038090,
"wiki_size" : 0,
"lfs_objects_size": 0,
"job_artifacts_size": 0
},

2
lib/api/entities.rb
View file

@ -303,6 +303,7 @@ module API
expose :commit_count
expose :storage_size
expose :repository_size
expose :wiki_size
expose :lfs_objects_size
expose :build_artifacts_size, as: :job_artifacts_size
end
@ -355,6 +356,7 @@ module API
with_options format_with: -> (value) { value.to_i } do
expose :storage_size
expose :repository_size
expose :wiki_size
expose :lfs_objects_size
expose :build_artifacts_size, as: :job_artifacts_size
end

18
lib/gitlab/background_migration/calculate_wiki_sizes.rb Normal file
View file

@ -0,0 +1,18 @@
# frozen_string_literal: true
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
class CalculateWikiSizes
def perform(start_id, stop_id)
::ProjectStatistics.where(wiki_size: nil)
.where(id: start_id..stop_id)
.includes(project: [:route, :group, namespace: [:owner]]).find_each do |statistics|
statistics.refresh!(only: [:wiki_size])
rescue => e
Rails.logger.error "Failed to update wiki statistics. id: #{statistics.id} message: #{e.message}"
end
end
end
end
end

2
locale/gitlab.pot
View file

@ -107,7 +107,7 @@ msgstr ""
msgid "%{commit_author_link} authored %{commit_timeago}"
msgstr ""
msgid "%{counter_repositories} repositories, %{counter_build_artifacts} build artifacts, %{counter_lfs_objects} LFS"
msgid "%{counter_repositories} repositories, %{counter_wikis} wikis, %{counter_build_artifacts} build artifacts, %{counter_lfs_objects} LFS"
msgstr ""
msgid "%{count} more"

2
spec/features/admin/admin_sees_project_statistics_spec.rb
View file

@ -15,7 +15,7 @@ describe "Admin > Admin sees project statistics" do
let(:project) { create(:project, :repository) }
it "shows project statistics" do
expect(page).to have_content("Storage: 0 Bytes (0 Bytes repositories, 0 Bytes build artifacts, 0 Bytes LFS)")
expect(page).to have_content("Storage: 0 Bytes (0 Bytes repositories, 0 Bytes wikis, 0 Bytes build artifacts, 0 Bytes LFS)")
end
end

3
spec/helpers/storage_helper_spec.rb
View file

@ -26,11 +26,12 @@ describe StorageHelper do
namespace: namespace,
statistics: build(:project_statistics,
repository_size: 10.kilobytes,
wiki_size: 10.bytes,
lfs_objects_size: 20.gigabytes,
build_artifacts_size: 30.megabytes))
end
let(:message) { '10 KB repositories, 30 MB build artifacts, 20 GB LFS' }
let(:message) { '10 KB repositories, 10 Bytes wikis, 30 MB build artifacts, 20 GB LFS' }
it 'works on ProjectStatistics' do
expect(helper.storage_counters_details(project.statistics)).to eq(message)

62
spec/lib/gitlab/background_migration/schedule_calculate_wiki_sizes_spec.rb Normal file
View file

@ -0,0 +1,62 @@
require 'spec_helper'
require Rails.root.join('db', 'post_migrate', '20190527194900_schedule_calculate_wiki_sizes.rb')
describe ScheduleCalculateWikiSizes, :migration, :sidekiq do
let(:migration_class) { Gitlab::BackgroundMigration::CalculateWikiSizes }
let(:migration_name) { migration_class.to_s.demodulize }
let(:namespaces) { table(:namespaces) }
let(:projects) { table(:projects) }
let(:project_statistics) { table(:project_statistics) }
context 'when missing wiki sizes exist' do
before do
namespaces.create!(id: 1, name: 'wiki-migration', path: 'wiki-migration')
projects.create!(id: 1, name: 'wiki-project-1', path: 'wiki-project-1', namespace_id: 1)
projects.create!(id: 2, name: 'wiki-project-2', path: 'wiki-project-2', namespace_id: 1)
projects.create!(id: 3, name: 'wiki-project-3', path: 'wiki-project-3', namespace_id: 1)
project_statistics.create!(id: 1, project_id: 1, namespace_id: 1, wiki_size: 1000)
project_statistics.create!(id: 2, project_id: 2, namespace_id: 1, wiki_size: nil)
project_statistics.create!(id: 3, project_id: 3, namespace_id: 1, wiki_size: nil)
end
it 'schedules a background migration' do
Sidekiq::Testing.fake! do
Timecop.freeze do
migrate!
expect(migration_name).to be_scheduled_delayed_migration(5.minutes, 2, 3)
expect(BackgroundMigrationWorker.jobs.size).to eq 1
end
end
end
it 'calculates missing wiki sizes' do
expect(project_statistics.find_by(id: 2).wiki_size).to be_nil
expect(project_statistics.find_by(id: 3).wiki_size).to be_nil
migrate!
expect(project_statistics.find_by(id: 2).wiki_size).not_to be_nil
expect(project_statistics.find_by(id: 3).wiki_size).not_to be_nil
end
end
context 'when missing wiki sizes do not exist' do
before do
namespaces.create!(id: 1, name: 'wiki-migration', path: 'wiki-migration')
projects.create!(id: 1, name: 'wiki-project-1', path: 'wiki-project-1', namespace_id: 1)
project_statistics.create!(id: 1, project_id: 1, namespace_id: 1, wiki_size: 1000)
end
it 'does not schedule a background migration' do
Sidekiq::Testing.fake! do
Timecop.freeze do
migrate!
expect(BackgroundMigrationWorker.jobs.size).to eq 0
end
end
end
end
end

6
spec/models/namespace_spec.rb
View file

@ -147,6 +147,7 @@ describe Namespace do
namespace: namespace,
statistics: build(:project_statistics,
repository_size: 101,
wiki_size: 505,
lfs_objects_size: 202,
build_artifacts_size: 303,
packages_size: 404))
@ -157,6 +158,7 @@ describe Namespace do
namespace: namespace,
statistics: build(:project_statistics,
repository_size: 10,
wiki_size: 50,
lfs_objects_size: 20,
build_artifacts_size: 30,
packages_size: 40))
@ -167,8 +169,9 @@ describe Namespace do
project2
statistics = described_class.with_statistics.find(namespace.id)
expect(statistics.storage_size).to eq 1110
expect(statistics.storage_size).to eq 1665
expect(statistics.repository_size).to eq 111
expect(statistics.wiki_size).to eq 555
expect(statistics.lfs_objects_size).to eq 222
expect(statistics.build_artifacts_size).to eq 333
expect(statistics.packages_size).to eq 444
@ -179,6 +182,7 @@ describe Namespace do
expect(statistics.storage_size).to eq 0
expect(statistics.repository_size).to eq 0
expect(statistics.wiki_size).to eq 0
expect(statistics.lfs_objects_size).to eq 0
expect(statistics.build_artifacts_size).to eq 0
expect(statistics.packages_size).to eq 0

68
spec/models/project_statistics_spec.rb
View file

@ -16,16 +16,18 @@ describe ProjectStatistics do
statistics.update!(
commit_count: 8.exabytes - 1,
repository_size: 2.exabytes,
wiki_size: 1.exabytes,
lfs_objects_size: 2.exabytes,
build_artifacts_size: 4.exabytes - 1
build_artifacts_size: 3.exabytes - 1
)
statistics.reload
expect(statistics.commit_count).to eq(8.exabytes - 1)
expect(statistics.repository_size).to eq(2.exabytes)
expect(statistics.wiki_size).to eq(1.exabytes)
expect(statistics.lfs_objects_size).to eq(2.exabytes)
expect(statistics.build_artifacts_size).to eq(4.exabytes - 1)
expect(statistics.build_artifacts_size).to eq(3.exabytes - 1)
expect(statistics.storage_size).to eq(8.exabytes - 1)
end
end
@ -33,6 +35,7 @@ describe ProjectStatistics do
describe '#total_repository_size' do
it "sums repository and LFS object size" do
statistics.repository_size = 2
statistics.wiki_size = 6
statistics.lfs_objects_size = 3
statistics.build_artifacts_size = 4
@ -40,10 +43,17 @@ describe ProjectStatistics do
end
end
describe '#wiki_size' do
it "is initialized with not null value" do
expect(statistics.wiki_size).to eq 0
end
end
describe '#refresh!' do
before do
allow(statistics).to receive(:update_commit_count)
allow(statistics).to receive(:update_repository_size)
allow(statistics).to receive(:update_wiki_size)
allow(statistics).to receive(:update_lfs_objects_size)
allow(statistics).to receive(:update_storage_size)
end
@ -56,6 +66,7 @@ describe ProjectStatistics do
it "sums all counters" do
expect(statistics).to have_received(:update_commit_count)
expect(statistics).to have_received(:update_repository_size)
expect(statistics).to have_received(:update_wiki_size)
expect(statistics).to have_received(:update_lfs_objects_size)
end
end
@ -69,6 +80,45 @@ describe ProjectStatistics do
expect(statistics).to have_received(:update_lfs_objects_size)
expect(statistics).not_to have_received(:update_commit_count)
expect(statistics).not_to have_received(:update_repository_size)
expect(statistics).not_to have_received(:update_wiki_size)
end
end
context 'without repositories' do
it 'does not crash' do
expect(project.repository.exists?).to be_falsey
expect(project.wiki.repository.exists?).to be_falsey
statistics.refresh!
expect(statistics).to have_received(:update_commit_count)
expect(statistics).to have_received(:update_repository_size)
expect(statistics).to have_received(:update_wiki_size)
expect(statistics.repository_size).to eq(0)
expect(statistics.commit_count).to eq(0)
expect(statistics.wiki_size).to eq(0)
end
end
context 'with deleted repositories' do
let(:project) { create(:project, :repository, :wiki_repo) }
before do
Gitlab::GitalyClient::StorageSettings.allow_disk_access do
FileUtils.rm_rf(project.repository.path)
FileUtils.rm_rf(project.wiki.repository.path)
end
end
it 'does not crash' do
statistics.refresh!
expect(statistics).to have_received(:update_commit_count)
expect(statistics).to have_received(:update_repository_size)
expect(statistics).to have_received(:update_wiki_size)
expect(statistics.repository_size).to eq(0)
expect(statistics.commit_count).to eq(0)
expect(statistics.wiki_size).to eq(0)
end
end
end
@ -95,6 +145,17 @@ describe ProjectStatistics do
end
end
describe '#update_wiki_size' do
before do
allow(project.wiki.repository).to receive(:size).and_return(34)
statistics.update_wiki_size
end
it "stores the size of the wiki" do
expect(statistics.wiki_size).to eq 34.megabytes
end
end
describe '#update_lfs_objects_size' do
let!(:lfs_object1) { create(:lfs_object, size: 23.megabytes) }
let!(:lfs_object2) { create(:lfs_object, size: 34.megabytes) }
@ -114,12 +175,13 @@ describe ProjectStatistics do
it "sums all storage counters" do
statistics.update!(
repository_size: 2,
wiki_size: 4,
lfs_objects_size: 3
)
statistics.reload
expect(statistics.storage_size).to eq 5
expect(statistics.storage_size).to eq 9
end
end

3
spec/requests/api/groups_spec.rb
View file

@ -90,8 +90,9 @@ describe API::Groups do
it "includes statistics if requested" do
attributes = {
storage_size: 702,
storage_size: 1158,
repository_size: 123,
wiki_size: 456,
lfs_objects_size: 234,
build_artifacts_size: 345
}.stringify_keys

12
spec/services/projects/update_statistics_service_spec.rb
View file

@ -17,19 +17,9 @@ describe Projects::UpdateStatisticsService do
end
end
context 'with an existing project without a repository' do
context 'with an existing project' do
let(:project) { create(:project) }
it 'does nothing' do
expect_any_instance_of(ProjectStatistics).not_to receive(:refresh!)
service.execute
end
end
context 'with an existing project with a repository' do
let(:project) { create(:project, :repository) }
it 'refreshes the project statistics' do
expect_any_instance_of(ProjectStatistics).to receive(:refresh!)
.with(only: statistics.map(&:to_sym))

5
spec/workers/project_cache_worker_spec.rb
View file

@ -25,10 +25,11 @@ describe ProjectCacheWorker do
end
context 'with an existing project without a repository' do
it 'does nothing' do
it 'updates statistics but does not refresh the method cashes' do
allow_any_instance_of(Repository).to receive(:exists?).and_return(false)
expect(worker).not_to receive(:update_statistics)
expect(worker).to receive(:update_statistics)
expect_any_instance_of(Repository).not_to receive(:refresh_method_caches)
worker.perform(project.id)
end