diff --git a/app/assets/stylesheets/pages/projects.scss b/app/assets/stylesheets/pages/projects.scss index aa83e5bdebc..2a41a045e71 100644 --- a/app/assets/stylesheets/pages/projects.scss +++ b/app/assets/stylesheets/pages/projects.scss @@ -754,6 +754,11 @@ } } +.repository-languages-bar { + height: 6px; + margin-bottom: 8px; +} + pre.light-well { border-color: $well-light-border; } diff --git a/app/helpers/repository_languages_helper.rb b/app/helpers/repository_languages_helper.rb new file mode 100644 index 00000000000..9a842cf5ce0 --- /dev/null +++ b/app/helpers/repository_languages_helper.rb @@ -0,0 +1,16 @@ +module RepositoryLanguagesHelper + def repository_languages_bar(languages) + return if languages.none? + + content_tag :div, class: 'progress repository-languages-bar' do + safe_join(languages.map { |lang| language_progress(lang) }) + end + end + + def language_progress(lang) + content_tag :div, nil, + class: "progress-bar has-tooltip", + style: "width: #{lang.share}%; background-color:#{lang.color}", + title: lang.name + end +end diff --git a/app/models/namespace.rb b/app/models/namespace.rb index c1dc2f55346..de06e080a7d 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -122,6 +122,7 @@ class Namespace < ActiveRecord::Base def to_param full_path end + alias_method :flipper_id, :to_param def human_name owner_name diff --git a/app/models/programming_language.rb b/app/models/programming_language.rb new file mode 100644 index 00000000000..400d6c407a7 --- /dev/null +++ b/app/models/programming_language.rb @@ -0,0 +1,4 @@ +class ProgrammingLanguage < ActiveRecord::Base + validates :name, presence: true + validates :color, allow_blank: false, color: true +end diff --git a/app/models/project.rb b/app/models/project.rb index da30d2fbb4f..af32afc08e2 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -192,6 +192,7 @@ class Project < ActiveRecord::Base has_many :hooks, class_name: 'ProjectHook' has_many :protected_branches has_many :protected_tags + has_many :repository_languages, -> { order "share DESC" } has_many :project_authorizations has_many :authorized_users, through: :project_authorizations, source: :user, class_name: 'User' diff --git a/app/models/repository.rb b/app/models/repository.rb index 50f42be661b..9a6281bc1f7 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -235,6 +235,12 @@ class Repository false end + def languages + return [] if empty? + + raw_repository.languages(root_ref) + end + # Makes sure a commit is kept around when Git garbage collection runs. # Git GC will delete commits from the repository that are no longer in any # branches or tags, but we want to keep some of these commits around, for @@ -432,6 +438,8 @@ class Repository # Runs code after a repository has been forked/imported. def after_import expire_content_cache + + DetectRepositoryLanguagesWorker.perform_async(project.id, project.owner.id) end # Runs code after a new commit has been pushed. diff --git a/app/models/repository_language.rb b/app/models/repository_language.rb new file mode 100644 index 00000000000..f467d4eafa3 --- /dev/null +++ b/app/models/repository_language.rb @@ -0,0 +1,12 @@ +class RepositoryLanguage < ActiveRecord::Base + belongs_to :project + belongs_to :programming_language + + default_scope { includes(:programming_language) } + + validates :project, presence: true + validates :share, inclusion: { in: 0..100, message: "The share of a lanuage is between 0 and 100" } + validates :programming_language, uniqueness: { scope: :project_id } + + delegate :name, :color, to: :programming_language +end diff --git a/app/services/git_push_service.rb b/app/services/git_push_service.rb index e3640e38bfa..637c1df4ad9 100644 --- a/app/services/git_push_service.rb +++ b/app/services/git_push_service.rb @@ -85,6 +85,8 @@ class GitPushService < BaseService types = Gitlab::FileDetector.types_in_paths(paths.to_a) end + + DetectRepositoryLanguagesWorker.perform_async(@project.id, current_user.id) else types = [] end diff --git a/app/services/projects/detect_repository_languages_service.rb b/app/services/projects/detect_repository_languages_service.rb new file mode 100644 index 00000000000..6669830ac8c --- /dev/null +++ b/app/services/projects/detect_repository_languages_service.rb @@ -0,0 +1,53 @@ +module Projects + class DetectRepositoryLanguagesService < BaseService + attr_reader :detected_repository_languages, :programming_languages + + def execute + repository_languages = project.repository_languages + detection = Gitlab::LanguageDetection.new(repository, repository_languages) + + matching_programming_languages = ensure_programming_languages(detection) + + RepositoryLanguage.transaction do + project.repository_languages.where(programming_language_id: detection.deletions).delete_all + + detection.updates.each do |update| + RepositoryLanguage + .arel_table.update_manager + .where(project_id: project.id) + .where(programming_language_id: update[:programming_language_id]) + .set(share: update[:share]) + end + + Gitlab::Database.bulk_insert( + RepositoryLanguage.table_name, + detection.insertions(matching_programming_languages) + ) + end + + project.repository_languages.reload + end + + private + + def ensure_programming_languages(detection) + existing_languages = ProgrammingLanguage.where(name: detection.languages) + return existing_languages if detection.languages.size == existing_languages.size + + missing_languages = detection.languages - existing_languages.map(&:name) + created_languages = missing_languages.map do |name| + create_language(name, detection.language_color(name)) + end + + existing_languages + created_languages + end + + def create_language(name, color) + ProgrammingLanguage.transaction do + ProgrammingLanguage.where(name: name).first_or_create(color: color) + end + rescue ActiveRecord::RecordNotUnique + retry + end + end +end diff --git a/app/views/projects/show.html.haml b/app/views/projects/show.html.haml index 803ecca48f7..e011851be78 100644 --- a/app/views/projects/show.html.haml +++ b/app/views/projects/show.html.haml @@ -18,10 +18,11 @@ = render "home_panel" - if can?(current_user, :download_code, @project) - %nav.project-stats{ class: container_class } + %nav.project-stats{ class: [container_class, ("limit-container-width" unless fluid_layout)] } = render 'stat_anchor_list', anchors: @project.statistics_anchors(show_auto_devops_callout: show_auto_devops_callout) = render 'stat_anchor_list', anchors: @project.statistics_buttons(show_auto_devops_callout: show_auto_devops_callout) - + - if Feature.enabled?(:repository_languages, @project.namespace.becomes(Namespace)) + = repository_languages_bar(@project.repository_languages) %div{ class: [container_class, ("limit-container-width" unless fluid_layout)] } - if @project.archived? diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index f2651cb54ec..e8b9999f83b 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -123,3 +123,4 @@ - repository_update_remote_mirror - create_note_diff_file - delete_diff_files +- detect_repository_languages diff --git a/app/workers/detect_repository_languages_worker.rb b/app/workers/detect_repository_languages_worker.rb new file mode 100644 index 00000000000..537b8fd5963 --- /dev/null +++ b/app/workers/detect_repository_languages_worker.rb @@ -0,0 +1,33 @@ +class DetectRepositoryLanguagesWorker + include ApplicationWorker + include ExceptionBacktrace + include ExclusiveLeaseGuard + + sidekiq_options retry: 1 + + LEASE_TIMEOUT = 300 + + attr_reader :project + + def perform(project_id, user_id) + @project = Project.find_by(id: project_id) + user = User.find_by(id: user_id) + return unless project && user + + return if Feature.disabled?(:repository_languages, project.namespace) + + try_obtain_lease do + ::Projects::DetectRepositoryLanguagesService.new(project, user).execute + end + end + + private + + def lease_timeout + LEASE_TIMEOUT + end + + def lease_key + "gitlab:detect_repository_languages:#{project.id}" + end +end diff --git a/changelogs/unreleased/zj-repository-languages.yml b/changelogs/unreleased/zj-repository-languages.yml new file mode 100644 index 00000000000..c42ba60be29 --- /dev/null +++ b/changelogs/unreleased/zj-repository-languages.yml @@ -0,0 +1,5 @@ +--- +title: Show repository languages for projects +merge_request: 19480 +author: +type: added diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index 3c85cd07d46..fb7738a5536 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -77,3 +77,4 @@ - [repository_remove_remote, 1] - [create_note_diff_file, 1] - [delete_diff_files, 1] + - [detect_repository_languages, 1] diff --git a/db/migrate/20180531185349_add_repository_languages.rb b/db/migrate/20180531185349_add_repository_languages.rb new file mode 100644 index 00000000000..bfcfb618c87 --- /dev/null +++ b/db/migrate/20180531185349_add_repository_languages.rb @@ -0,0 +1,28 @@ +class AddRepositoryLanguages < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def up + create_table(:programming_languages) do |t| + t.string :name, null: false + t.string :color, null: false + t.datetime_with_timezone :created_at, null: false + end + + create_table(:repository_languages, id: false) do |t| + t.references :project, null: false, foreign_key: { on_delete: :cascade } + t.references :programming_language, null: false + t.float :share, null: false + end + + add_index :programming_languages, :name, unique: true + add_index :repository_languages, [:project_id, :programming_language_id], + unique: true, name: "index_repository_languages_on_project_and_languages_id" + end + + def down + drop_table :repository_languages + drop_table :programming_languages + end +end diff --git a/db/schema.rb b/db/schema.rb index 6f3433c1003..769baa825a5 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -1502,6 +1502,14 @@ ActiveRecord::Schema.define(version: 20180726172057) do add_index "personal_access_tokens", ["token"], name: "index_personal_access_tokens_on_token", unique: true, using: :btree add_index "personal_access_tokens", ["user_id"], name: "index_personal_access_tokens_on_user_id", using: :btree + create_table "programming_languages", force: :cascade do |t| + t.string "name", null: false + t.string "color", null: false + t.datetime_with_timezone "created_at", null: false + end + + add_index "programming_languages", ["name"], name: "index_programming_languages_on_name", unique: true, using: :btree + create_table "project_authorizations", id: false, force: :cascade do |t| t.integer "user_id", null: false t.integer "project_id", null: false @@ -1788,6 +1796,14 @@ ActiveRecord::Schema.define(version: 20180726172057) do add_index "remote_mirrors", ["last_successful_update_at"], name: "index_remote_mirrors_on_last_successful_update_at", using: :btree add_index "remote_mirrors", ["project_id"], name: "index_remote_mirrors_on_project_id", using: :btree + create_table "repository_languages", id: false, force: :cascade do |t| + t.integer "project_id", null: false + t.integer "programming_language_id", null: false + t.float "share", null: false + end + + add_index "repository_languages", ["project_id", "programming_language_id"], name: "index_repository_languages_on_project_and_languages_id", unique: true, using: :btree + create_table "resource_label_events", id: :bigserial, force: :cascade do |t| t.integer "action", null: false t.integer "issue_id" @@ -2359,6 +2375,7 @@ ActiveRecord::Schema.define(version: 20180726172057) do add_foreign_key "push_event_payloads", "events", name: "fk_36c74129da", on_delete: :cascade add_foreign_key "releases", "projects", name: "fk_47fe2a0596", on_delete: :cascade add_foreign_key "remote_mirrors", "projects", on_delete: :cascade + add_foreign_key "repository_languages", "projects", on_delete: :cascade add_foreign_key "resource_label_events", "issues", on_delete: :cascade add_foreign_key "resource_label_events", "labels", on_delete: :nullify add_foreign_key "resource_label_events", "merge_requests", on_delete: :cascade diff --git a/lib/feature.rb b/lib/feature.rb index d27b2b0e72f..09c5ef3ad94 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -46,6 +46,10 @@ class Feature get(key).enabled?(thing) end + def disabled?(key, thing = nil) + !enabled?(key, thing) + end + def enable(key, thing = true) get(key).enable(thing) end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index da3667faf7a..f69f98a78a3 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -107,6 +107,7 @@ excluded_attributes: - :storage_version - :remote_mirror_available_overridden - :description_html + - :repository_languages snippets: - :expired_at merge_request_diff: diff --git a/lib/gitlab/language_detection.rb b/lib/gitlab/language_detection.rb new file mode 100644 index 00000000000..a41435fdb79 --- /dev/null +++ b/lib/gitlab/language_detection.rb @@ -0,0 +1,68 @@ +module Gitlab + class LanguageDetection + MAX_LANGUAGES = 5 + + def initialize(repository, repository_languages) + @repository = repository + @repository_languages = repository_languages + end + + def languages + detection.keys + end + + def language_color(name) + detection.dig(name, :color) + end + + # Newly detected languages, returned in a structure accepted by + # Gitlab::Database.bulk_insert + def insertions(programming_languages) + lang_to_id = programming_languages.map { |p| [p.name, p.id] }.to_h + + (languages - previous_language_names).map do |new_lang| + { + project_id: @repository.project.id, + share: detection[new_lang][:value], + programming_language_id: lang_to_id[new_lang] + } + end + end + + # updates analyses which records only require updating of their share + def updates + to_update = @repository_languages.select do |lang| + detection.key?(lang.name) && detection[lang.name][:value] != lang.share + end + + to_update.map do |lang| + { programming_language_id: lang.programming_language_id, share: detection[lang.name][:value] } + end + end + + # Returns the ids of the programming languages that do not occur in the detection + # as current repository languages + def deletions + @repository_languages.map do |repo_lang| + next if detection.key?(repo_lang.name) + + repo_lang.programming_language_id + end.compact + end + + private + + def previous_language_names + @previous_language_names ||= @repository_languages.map(&:name) + end + + def detection + @detection ||= + @repository + .languages + .first(MAX_LANGUAGES) + .map { |l| [l[:label], l] } + .to_h + end + end +end diff --git a/spec/factories/programming_languages.rb b/spec/factories/programming_languages.rb new file mode 100644 index 00000000000..d3511d42b6c --- /dev/null +++ b/spec/factories/programming_languages.rb @@ -0,0 +1,6 @@ +FactoryBot.define do + factory :programming_language do + name 'Ruby' + color '#123456' + end +end diff --git a/spec/factories/repository_languages.rb b/spec/factories/repository_languages.rb new file mode 100644 index 00000000000..1757ba6766c --- /dev/null +++ b/spec/factories/repository_languages.rb @@ -0,0 +1,7 @@ +FactoryBot.define do + factory :repository_language do + project + programming_language + share 98.5 + end +end diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index c175dc1e4dd..b3e3ead9c5e 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -297,6 +297,7 @@ project: - settings - ci_cd_settings - import_export_upload +- repository_languages award_emoji: - awardable - user diff --git a/spec/lib/gitlab/language_detection_spec.rb b/spec/lib/gitlab/language_detection_spec.rb new file mode 100644 index 00000000000..9636fbd401b --- /dev/null +++ b/spec/lib/gitlab/language_detection_spec.rb @@ -0,0 +1,85 @@ +require 'spec_helper' + +describe Gitlab::LanguageDetection do + set(:project) { create(:project, :repository) } + set(:ruby) { create(:programming_language, name: 'Ruby') } + set(:haskell) { create(:programming_language, name: 'Haskell') } + let(:repository) { project.repository } + let(:detection) do + [{ value: 66.63, label: "Ruby", color: "#701516", highlight: "#701516" }, + { value: 12.96, label: "JavaScript", color: "#f1e05a", highlight: "#f1e05a" }, + { value: 7.9, label: "Elixir", color: "#e34c26", highlight: "#e34c26" }, + { value: 2.51, label: "CoffeeScript", color: "#244776", highlight: "#244776" }, + { value: 1.51, label: "Go", color: "#2a4776", highlight: "#244776" }, + { value: 1.1, label: "MepmepLang", color: "#2a4776", highlight: "#244776" }] + end + let(:repository_languages) do + [RepositoryLanguage.new(share: 10, programming_language: ruby)] + end + + subject { described_class.new(repository, repository_languages) } + + before do + allow(repository).to receive(:languages).and_return(detection) + end + + describe '#languages' do + it 'returns the language names' do + expect(subject.languages).to eq(%w[Ruby JavaScript Elixir CoffeeScript Go]) + end + end + + describe '#insertions' do + let(:programming_languages) { [ruby, haskell] } + let(:detection) do + [{ value: 10, label: haskell.name, color: haskell.color }] + end + + it 'only includes new languages' do + insertions = subject.insertions(programming_languages) + + expect(insertions).not_to be_empty + expect(insertions.first[:project_id]).to be(project.id) + expect(insertions.first[:programming_language_id]).to be(haskell.id) + expect(insertions.first[:share]).to be(10) + end + end + + describe '#updates' do + it 'updates the share of languages' do + first_update = subject.updates.first + + expect(first_update).not_to be_nil + expect(first_update[:programming_language_id]).to eq(ruby.id) + expect(first_update[:share]).to eq(66.63) + end + + it 'does not include languages to be removed' do + ids = subject.updates.map { |h| h[:programming_language_id] } + + expect(ids).not_to include(haskell.id) + end + + context 'when silent writes occur' do + let(:repository_languages) do + [RepositoryLanguage.new(share: 66.63, programming_language: ruby)] + end + + it "doesn't include them in the result" do + expect(subject.updates).to be_empty + end + end + end + + describe '#deletions' do + let(:repository_languages) do + [RepositoryLanguage.new(share: 10, programming_language: ruby), + RepositoryLanguage.new(share: 5, programming_language: haskell)] + end + + it 'lists undetected languages' do + expect(subject.deletions).not_to be_empty + expect(subject.deletions).to include(haskell.id) + end + end +end diff --git a/spec/models/programming_language_spec.rb b/spec/models/programming_language_spec.rb new file mode 100644 index 00000000000..99cd358f863 --- /dev/null +++ b/spec/models/programming_language_spec.rb @@ -0,0 +1,11 @@ +require 'spec_helper' + +describe ProgrammingLanguage do + it { is_expected.to respond_to(:name) } + it { is_expected.to respond_to(:color) } + + it { is_expected.to validate_presence_of(:name) } + it { is_expected.to allow_value("#000000").for(:color) } + it { is_expected.not_to allow_value("000000").for(:color) } + it { is_expected.not_to allow_value("#0z0000").for(:color) } +end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index ef4167a3912..340d2d95500 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -69,6 +69,7 @@ describe Project do it { is_expected.to have_many(:pages_domains) } it { is_expected.to have_many(:labels).class_name('ProjectLabel') } it { is_expected.to have_many(:users_star_projects) } + it { is_expected.to have_many(:repository_languages) } it { is_expected.to have_many(:environments) } it { is_expected.to have_many(:deployments) } it { is_expected.to have_many(:todos) } diff --git a/spec/models/repository_language_spec.rb b/spec/models/repository_language_spec.rb new file mode 100644 index 00000000000..e2e4beb512f --- /dev/null +++ b/spec/models/repository_language_spec.rb @@ -0,0 +1,16 @@ +require 'spec_helper' + +describe RepositoryLanguage do + let(:repository_language) { build(:repository_language) } + + describe 'associations' do + it { is_expected.to belong_to(:project) } + it { is_expected.to belong_to(:programming_language) } + end + + describe 'validations' do + it { is_expected.to allow_value(0).for(:share) } + it { is_expected.to allow_value(100.0).for(:share) } + it { is_expected.not_to allow_value(100.1).for(:share) } + end +end diff --git a/spec/services/git_push_service_spec.rb b/spec/services/git_push_service_spec.rb index 657afb2b2b5..a3c9a660c2f 100644 --- a/spec/services/git_push_service_spec.rb +++ b/spec/services/git_push_service_spec.rb @@ -3,8 +3,8 @@ require 'spec_helper' describe GitPushService, services: true do include RepoHelpers - let(:user) { create(:user) } - let(:project) { create(:project, :repository) } + set(:user) { create(:user) } + set(:project) { create(:project, :repository) } let(:blankrev) { Gitlab::Git::BLANK_SHA } let(:oldrev) { sample_commit.parent_id } let(:newrev) { sample_commit.id } diff --git a/spec/services/projects/detect_repository_languages_service_spec.rb b/spec/services/projects/detect_repository_languages_service_spec.rb new file mode 100644 index 00000000000..f90d558938f --- /dev/null +++ b/spec/services/projects/detect_repository_languages_service_spec.rb @@ -0,0 +1,54 @@ +require 'spec_helper' + +describe Projects::DetectRepositoryLanguagesService, :clean_gitlab_redis_shared_state do + set(:project) { create(:project, :repository) } + + subject { described_class.new(project, project.owner) } + + before do + allow(Feature).to receive(:disabled?).and_return(false) + end + + describe '#execute' do + context 'without previous detection' do + it 'inserts new programming languages in the database' do + subject.execute + + expect(ProgrammingLanguage.exists?(name: 'Ruby')).to be(true) + expect(ProgrammingLanguage.count).to be(4) + end + + it 'inserts the repository langauges' do + names = subject.execute.map(&:name) + + expect(names).to eq(%w[Ruby JavaScript HTML CoffeeScript]) + end + end + + context 'with a previous detection' do + before do + subject.execute + + allow(project.repository).to receive(:languages).and_return( + [{ value: 99.63, label: "Ruby", color: "#701516", highlight: "#701516" }, + { value: 0.3, label: "D", color: "#701516", highlight: "#701516" }] + ) + end + + it 'updates the repository languages' do + repository_languages = subject.execute.map(&:name) + + expect(repository_languages).to eq(%w[Ruby D]) + end + end + + context 'when no repository exists' do + set(:project) { create(:project) } + + it 'has no languages' do + expect(subject.execute).to be_empty + expect(project.repository_languages).to be_empty + end + end + end +end diff --git a/spec/workers/detect_repository_languages_worker_spec.rb b/spec/workers/detect_repository_languages_worker_spec.rb new file mode 100644 index 00000000000..ff3878fbc8e --- /dev/null +++ b/spec/workers/detect_repository_languages_worker_spec.rb @@ -0,0 +1,32 @@ +require 'spec_helper' + +describe DetectRepositoryLanguagesWorker do + set(:project) { create(:project) } + let(:user) { project.owner } + + subject { described_class.new } + + describe '#perform' do + it 'calls de DetectRepositoryLanguages service' do + service = double + allow(::Projects::DetectRepositoryLanguagesService).to receive(:new).and_return(service) + expect(service).to receive(:execute) + + subject.perform(project.id, user.id) + end + + context 'when invalid ids are used' do + it 'does not raise when the project could not be found' do + expect do + subject.perform(-1, user.id) + end.not_to raise_error + end + + it 'does not raise when the user could not be found' do + expect do + subject.perform(project.id, -1) + end.not_to raise_error + end + end + end +end