Add basic project extraction

To allow project filtering

Prepare summary for accepting multiple groups

Modify deploys group summary class

Add filtering by project name in issues summary

Fix rubocop offences

Add changelog entry

Change name to id in project filtering

Fix rebase problem

Add project extraction

app/models/cycle_analytics/group_level.rb

@@ -13,7 +13,8 @@ module CycleAnalytics
     def summary
       @summary ||= ::Gitlab::CycleAnalytics::GroupStageSummary.new(group,
                                                               from: options[:from],
-                                                              current_user: options[:current_user]).data
+                                                              current_user: options[:current_user],
+                                                              options: options).data
     end
 
     def permissions(*)

changelogs/unreleased/adjust-group-level-analytics-to-accept-multiple-ids.yml

@@ -0,0 +1,5 @@
+---
+title: Adjust group level analytics to accept multiple ids
+merge_request: 30468
+author:
+type: added

db/schema.rb

@@ -63,7 +63,6 @@ ActiveRecord::Schema.define(version: 2019_07_15_114644) do
     t.datetime "updated_at"
     t.string "home_page_url"
     t.integer "default_branch_protection", default: 2
-    t.text "help_text"
     t.text "restricted_visibility_levels"
     t.boolean "version_check_enabled", default: true
     t.integer "max_attachment_size", default: 10, null: false
@@ -105,8 +104,6 @@ ActiveRecord::Schema.define(version: 2019_07_15_114644) do
     t.integer "container_registry_token_expire_delay", default: 5
     t.text "after_sign_up_text"
     t.boolean "user_default_external", default: false, null: false
-    t.boolean "elasticsearch_indexing", default: false, null: false
-    t.boolean "elasticsearch_search", default: false, null: false
     t.string "repository_storages", default: "default"
     t.string "enabled_git_access_protocol"
     t.boolean "domain_blacklist_enabled", default: false
@@ -128,37 +125,18 @@ ActiveRecord::Schema.define(version: 2019_07_15_114644) do
     t.boolean "html_emails_enabled", default: true
     t.string "plantuml_url"
     t.boolean "plantuml_enabled"
-    t.integer "shared_runners_minutes", default: 0, null: false
-    t.bigint "repository_size_limit", default: 0
     t.integer "terminal_max_session_time", default: 0, null: false
     t.integer "unique_ips_limit_per_user"
     t.integer "unique_ips_limit_time_window"
     t.boolean "unique_ips_limit_enabled", default: false, null: false
     t.string "default_artifacts_expire_in", default: "0", null: false
-    t.string "elasticsearch_url", default: "http://localhost:9200"
-    t.boolean "elasticsearch_aws", default: false, null: false
-    t.string "elasticsearch_aws_region", default: "us-east-1"
-    t.string "elasticsearch_aws_access_key"
-    t.string "elasticsearch_aws_secret_access_key"
-    t.integer "geo_status_timeout", default: 10
     t.string "uuid"
     t.decimal "polling_interval_multiplier", default: "1.0", null: false
-    t.boolean "elasticsearch_experimental_indexer"
     t.integer "cached_markdown_version"
-    t.boolean "check_namespace_plan", default: false, null: false
-    t.integer "mirror_max_delay", default: 300, null: false
-    t.integer "mirror_max_capacity", default: 100, null: false
-    t.integer "mirror_capacity_threshold", default: 50, null: false
     t.boolean "prometheus_metrics_enabled", default: true, null: false
-    t.boolean "authorized_keys_enabled", default: true, null: false
     t.boolean "help_page_hide_commercial_content", default: false
     t.string "help_page_support_url"
-    t.boolean "slack_app_enabled", default: false
-    t.string "slack_app_id"
-    t.string "slack_app_secret"
-    t.string "slack_app_verification_token"
     t.integer "performance_bar_allowed_group_id"
-    t.boolean "allow_group_owners_to_manage_ldap", default: true, null: false
     t.boolean "hashed_storage_enabled", default: true, null: false
     t.boolean "project_export_enabled", default: true, null: false
     t.boolean "auto_devops_enabled", default: true, null: false
@@ -171,38 +149,22 @@ ActiveRecord::Schema.define(version: 2019_07_15_114644) do
     t.boolean "throttle_authenticated_web_enabled", default: false, null: false
     t.integer "throttle_authenticated_web_requests_per_period", default: 7200, null: false
     t.integer "throttle_authenticated_web_period_in_seconds", default: 3600, null: false
+    t.boolean "password_authentication_enabled_for_web"
+    t.boolean "password_authentication_enabled_for_git", default: true, null: false
     t.integer "gitaly_timeout_default", default: 55, null: false
     t.integer "gitaly_timeout_medium", default: 30, null: false
     t.integer "gitaly_timeout_fast", default: 10, null: false
-    t.boolean "mirror_available", default: true, null: false
+    t.boolean "authorized_keys_enabled", default: true, null: false
-    t.boolean "password_authentication_enabled_for_web"
-    t.boolean "password_authentication_enabled_for_git", default: true, null: false
     t.string "auto_devops_domain"
-    t.boolean "external_authorization_service_enabled", default: false, null: false
-    t.string "external_authorization_service_url"
-    t.string "external_authorization_service_default_label"
     t.boolean "pages_domain_verification_enabled", default: true, null: false
     t.string "user_default_internal_regex"
     t.boolean "allow_local_requests_from_hooks_and_services", default: false, null: false
-    t.float "external_authorization_service_timeout", default: 0.5
-    t.text "external_auth_client_cert"
-    t.text "encrypted_external_auth_client_key"
-    t.string "encrypted_external_auth_client_key_iv"
-    t.string "encrypted_external_auth_client_key_pass"
-    t.string "encrypted_external_auth_client_key_pass_iv"
-    t.string "email_additional_text"
     t.boolean "enforce_terms", default: false
-    t.integer "file_template_project_id"
+    t.boolean "mirror_available", default: true, null: false
-    t.boolean "pseudonymizer_enabled", default: false, null: false
     t.boolean "hide_third_party_offers", default: false, null: false
-    t.boolean "snowplow_enabled", default: false, null: false
-    t.string "snowplow_collector_uri"
-    t.string "snowplow_site_id"
-    t.string "snowplow_cookie_domain"
     t.boolean "instance_statistics_visibility_private", default: false, null: false
     t.boolean "web_ide_clientside_preview_enabled", default: false, null: false
     t.boolean "user_show_add_ssh_key_message", default: true, null: false
-    t.integer "custom_project_templates_group_id"
     t.integer "usage_stats_set_by_user_id"
     t.integer "receive_max_input_size"
     t.integer "diff_max_patch_bytes", default: 102400, null: false
@@ -212,21 +174,59 @@ ActiveRecord::Schema.define(version: 2019_07_15_114644) do
     t.string "runners_registration_token_encrypted"
     t.integer "local_markdown_version", default: 0, null: false
     t.integer "first_day_of_week", default: 0, null: false
-    t.boolean "elasticsearch_limit_indexing", default: false, null: false
     t.integer "default_project_creation", default: 2, null: false
+    t.boolean "external_authorization_service_enabled", default: false, null: false
+    t.string "external_authorization_service_url"
+    t.string "external_authorization_service_default_label"
+    t.float "external_authorization_service_timeout", default: 0.5
+    t.text "external_auth_client_cert"
+    t.text "encrypted_external_auth_client_key"
+    t.string "encrypted_external_auth_client_key_iv"
+    t.string "encrypted_external_auth_client_key_pass"
+    t.string "encrypted_external_auth_client_key_pass_iv"
     t.string "lets_encrypt_notification_email"
     t.boolean "lets_encrypt_terms_of_service_accepted", default: false, null: false
-    t.string "geo_node_allowed_ips", default: "0.0.0.0/0, ::/0"
     t.integer "elasticsearch_shards", default: 5, null: false
     t.integer "elasticsearch_replicas", default: 1, null: false
     t.text "encrypted_lets_encrypt_private_key"
     t.text "encrypted_lets_encrypt_private_key_iv"
-    t.string "required_instance_ci_template"
     t.boolean "dns_rebinding_protection_enabled", default: true, null: false
     t.boolean "default_project_deletion_protection", default: false, null: false
-    t.boolean "grafana_enabled", default: false, null: false
+    t.text "help_text"
+    t.boolean "elasticsearch_indexing", default: false, null: false
+    t.boolean "elasticsearch_search", default: false, null: false
+    t.integer "shared_runners_minutes", default: 0, null: false
+    t.bigint "repository_size_limit", default: 0
+    t.string "elasticsearch_url", default: "http://localhost:9200"
+    t.boolean "elasticsearch_aws", default: false, null: false
+    t.string "elasticsearch_aws_region", default: "us-east-1"
+    t.string "elasticsearch_aws_access_key"
+    t.string "elasticsearch_aws_secret_access_key"
+    t.integer "geo_status_timeout", default: 10
+    t.boolean "elasticsearch_experimental_indexer"
+    t.boolean "check_namespace_plan", default: false, null: false
+    t.integer "mirror_max_delay", default: 300, null: false
+    t.integer "mirror_max_capacity", default: 100, null: false
+    t.integer "mirror_capacity_threshold", default: 50, null: false
+    t.boolean "slack_app_enabled", default: false
+    t.string "slack_app_id"
+    t.string "slack_app_secret"
+    t.string "slack_app_verification_token"
+    t.boolean "allow_group_owners_to_manage_ldap", default: true, null: false
+    t.string "email_additional_text"
+    t.integer "file_template_project_id"
+    t.boolean "pseudonymizer_enabled", default: false, null: false
+    t.boolean "snowplow_enabled", default: false, null: false
+    t.string "snowplow_collector_uri"
+    t.string "snowplow_site_id"
+    t.string "snowplow_cookie_domain"
+    t.integer "custom_project_templates_group_id"
+    t.boolean "elasticsearch_limit_indexing", default: false, null: false
+    t.string "geo_node_allowed_ips", default: "0.0.0.0/0, ::/0"
+    t.string "required_instance_ci_template"
     t.boolean "lock_memberships_to_ldap", default: false, null: false
     t.boolean "time_tracking_limit_to_hours", default: false, null: false
+    t.boolean "grafana_enabled", default: false, null: false
     t.string "grafana_url", default: "/-/grafana", null: false
     t.index ["custom_project_templates_group_id"], name: "index_application_settings_on_custom_project_templates_group_id", using: :btree
     t.index ["file_template_project_id"], name: "index_application_settings_on_file_template_project_id", using: :btree
@@ -407,10 +407,10 @@ ActiveRecord::Schema.define(version: 2019_07_15_114644) do
     t.integer "project_id"
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
-    t.string "name", default: "Development", null: false
-    t.integer "milestone_id"
     t.integer "group_id"
+    t.integer "milestone_id"
     t.integer "weight"
+    t.string "name", default: "Development", null: false
     t.index ["group_id"], name: "index_boards_on_group_id", using: :btree
     t.index ["milestone_id"], name: "index_boards_on_milestone_id", using: :btree
     t.index ["project_id"], name: "index_boards_on_project_id", using: :btree
@@ -611,7 +611,7 @@ ActiveRecord::Schema.define(version: 2019_07_15_114644) do
     t.index ["pipeline_id"], name: "index_ci_pipeline_chat_data_on_pipeline_id", unique: true, using: :btree
   end
 
-  create_table "ci_pipeline_schedule_variables", force: :cascade do |t|
+  create_table "ci_pipeline_schedule_variables", id: :serial, force: :cascade do |t|
     t.string "key", null: false
     t.text "value"
     t.text "encrypted_value"
@@ -1865,7 +1865,7 @@ ActiveRecord::Schema.define(version: 2019_07_15_114644) do
     t.index ["user_id"], name: "index_members_on_user_id", using: :btree
   end
 
-  create_table "merge_request_assignees", force: :cascade do |t|
+  create_table "merge_request_assignees", id: :serial, force: :cascade do |t|
     t.integer "user_id", null: false
     t.integer "merge_request_id", null: false
     t.index ["merge_request_id", "user_id"], name: "index_merge_request_assignees_on_merge_request_id_and_user_id", unique: true, using: :btree
@@ -2101,8 +2101,8 @@ ActiveRecord::Schema.define(version: 2019_07_15_114644) do
     t.integer "cached_markdown_version"
     t.string "runners_token"
     t.string "runners_token_encrypted"
-    t.integer "project_creation_level"
     t.boolean "auto_devops_enabled"
+    t.integer "project_creation_level"
     t.datetime_with_timezone "last_ci_minutes_notification_at"
     t.integer "custom_project_templates_group_id"
     t.integer "file_template_project_id"
@@ -2534,7 +2534,7 @@ ActiveRecord::Schema.define(version: 2019_07_15_114644) do
     t.index ["project_id"], name: "index_project_import_data_on_project_id", using: :btree
   end
 
-  create_table "project_incident_management_settings", primary_key: "project_id", id: :serial, force: :cascade do |t|
+  create_table "project_incident_management_settings", primary_key: "project_id", id: :integer, default: nil, force: :cascade do |t|
     t.boolean "create_issue", default: true, null: false
     t.boolean "send_email", default: false, null: false
     t.text "issue_template_key"
@@ -2885,6 +2885,7 @@ ActiveRecord::Schema.define(version: 2019_07_15_114644) do
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.index ["path"], name: "index_redirect_routes_on_path", unique: true, using: :btree
+    t.index ["path"], name: "index_redirect_routes_on_path_text_pattern_ops", using: :btree, opclasses: {"path"=>"varchar_pattern_ops"}
     t.index ["source_type", "source_id"], name: "index_redirect_routes_on_source_type_and_source_id", using: :btree
   end
 
@@ -3304,6 +3305,7 @@ ActiveRecord::Schema.define(version: 2019_07_15_114644) do
     t.string "timezone"
     t.boolean "time_display_relative"
     t.boolean "time_format_in_24h"
+    t.string "timezone_name"
     t.integer "epic_notes_filter", limit: 2, default: 0, null: false
     t.string "epics_sort"
     t.integer "roadmap_epics_state"

lib/gitlab/cycle_analytics/base_data_extraction.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module CycleAnalytics
+    module BaseDataExtraction
+      private
+
+      def projects
+        group ? extract_projects(options) : [project]
+      end
+
+      def group
+        @group ||= options.fetch(:group, nil)
+      end
+
+      def project
+        @project ||= options.fetch(:project, nil)
+      end
+
+      def extract_projects(options)
+        projects = Project.inside_path(group.full_path)
+        projects = projects.where(id: options[:projects]) if options[:projects]
+        projects
+      end
+    end
+  end
+end

lib/gitlab/cycle_analytics/base_event_fetcher.rb

@@ -4,6 +4,7 @@ module Gitlab
   module CycleAnalytics
     class BaseEventFetcher
       include BaseQuery
+      include BaseDataExtraction
 
       attr_reader :projections, :query, :stage, :order, :options
 
@@ -73,18 +74,6 @@ module Gitlab
       def serialization_context
         {}
       end
-
-      def projects
-        group ? Project.inside_path(group.full_path) : [project]
-      end
-
-      def group
-        @group ||= options.fetch(:group, nil)
-      end
-
-      def project
-        @project ||= options.fetch(:project, nil)
-      end
     end
   end
 end

lib/gitlab/cycle_analytics/base_stage.rb

@@ -4,6 +4,7 @@ module Gitlab
   module CycleAnalytics
     class BaseStage
       include BaseQuery
+      include BaseDataExtraction
 
       attr_reader :options
 
@@ -77,18 +78,6 @@ module Gitlab
       def event_options
         options.merge(start_time_attrs: start_time_attrs, end_time_attrs: end_time_attrs)
       end
-
-      def projects
-        group ? Project.inside_path(group.full_path) : [project]
-      end
-
-      def group
-        @group ||= options.fetch(:group, nil)
-      end
-
-      def project
-        @project ||= options.fetch(:project, nil)
-      end
     end
   end
 end

lib/gitlab/cycle_analytics/group_stage_summary.rb

@@ -3,15 +3,16 @@
 module Gitlab
   module CycleAnalytics
     class GroupStageSummary
-      def initialize(group, from:, current_user:)
+      def initialize(group, from:, current_user:, options:)
         @group = group
         @from = from
         @current_user = current_user
+        @options = options
       end
 
       def data
-        [serialize(Summary::Group::Issue.new(group: @group, from: @from, current_user: @current_user)),
+        [serialize(Summary::Group::Issue.new(group: @group, from: @from, current_user: @current_user, options: @options)),
-         serialize(Summary::Group::Deploy.new(group: @group, from: @from))]
+         serialize(Summary::Group::Deploy.new(group: @group, from: @from, options: @options))]
       end
 
       private

lib/gitlab/cycle_analytics/summary/group/base.rb

@@ -5,9 +5,10 @@ module Gitlab
     module Summary
       module Group
         class Base
-          def initialize(group:, from:)
+          def initialize(group:, from:, options:)
             @group = group
             @from = from
+            @options = options
           end
 
           def title

lib/gitlab/cycle_analytics/summary/group/deploy.rb

@@ -10,15 +10,19 @@ module Gitlab
           end
 
           def value
-            @value ||= Deployment.joins(:project)
+            @value ||= find_deployments
-              .where(projects: { id: projects })
-              .where("deployments.created_at > ?", @from)
-              .success
-              .count
           end
 
           private
 
+          def find_deployments
+            deployments = Deployment.joins(:project)
+              .where(projects: { id: projects })
+              .where("deployments.created_at > ?", @from)
+            deployments = deployments.where(projects: { id: @options[:projects] }) if @options[:projects]
+            deployments.success.count
+          end
+
           def projects
             Project.inside_path(@group.full_path).ids
           end

lib/gitlab/cycle_analytics/summary/group/issue.rb

@@ -5,10 +5,11 @@ module Gitlab
     module Summary
       module Group
         class Issue < Group::Base
-          def initialize(group:, from:, current_user:)
+          def initialize(group:, from:, current_user:, options:)
             @group = group
             @from = from
             @current_user = current_user
+            @options = options
           end
 
           def title
@@ -16,7 +17,15 @@ module Gitlab
           end
 
           def value
-            @value ||= IssuesFinder.new(@current_user, group_id: @group.id, include_subgroups: true, created_after: @from).execute.count
+            @value ||= find_issues
+          end
+
+          private
+
+          def find_issues
+            issues = IssuesFinder.new(@current_user, group_id: @group.id, include_subgroups: true, created_after: @from).execute
+            issues = issues.where(projects: { id: @options[:projects] }) if @options[:projects]
+            issues.count
           end
         end
       end

spec/lib/gitlab/cycle_analytics/group_stage_summary_spec.rb

@@ -8,7 +8,7 @@ describe Gitlab::CycleAnalytics::GroupStageSummary do
   let(:from) { 1.day.ago }
   let(:user) { create(:user, :admin) }
 
-  subject { described_class.new(group, from: Time.now, current_user: user).data }
+  subject { described_class.new(group, from: Time.now, current_user: user, options: {}).data }
 
   describe "#new_issues" do
     context 'with from date' do
@@ -32,6 +32,18 @@ describe Gitlab::CycleAnalytics::GroupStageSummary do
           expect(subject.first[:value]).to eq(3)
         end
       end
+
+      context 'with projects specified in options' do
+        before do
+          Timecop.freeze(5.days.from_now) { create(:issue, project: create(:project, namespace: group)) }
+        end
+
+        subject { described_class.new(group, from: Time.now, current_user: user, options: { projects: [project.id, project_2.id] }).data }
+
+        it 'finds issues from those projects' do
+          expect(subject.first[:value]).to eq(2)
+        end
+      end
     end
 
     context 'with other projects' do
@@ -71,6 +83,20 @@ describe Gitlab::CycleAnalytics::GroupStageSummary do
           expect(subject.second[:value]).to eq(3)
         end
       end
+
+      context 'with projects specified in options' do
+        before do
+          Timecop.freeze(5.days.from_now) do
+            create(:deployment, :success, project: create(:project, :repository, namespace: group, name: 'not_applicable'))
+          end
+        end
+
+        subject { described_class.new(group, from: Time.now, current_user: user, options: { projects: [project.id, project_2.id] }).data }
+
+        it 'shows deploys from those projects' do
+          expect(subject.second[:value]).to eq(2)
+        end
+      end
     end
 
     context 'with other projects' do
@@ -84,5 +110,6 @@ describe Gitlab::CycleAnalytics::GroupStageSummary do
         expect(subject.second[:value]).to eq(0)
       end
     end
+
   end
 end

spec/lib/gitlab/cycle_analytics/issue_stage_spec.rb

@@ -71,6 +71,29 @@ describe Gitlab::CycleAnalytics::IssueStage do
       end
     end
 
+    context 'when only part of projects is chosen' do
+      let(:stage) { described_class.new(options: { from: 2.days.ago, current_user: user, group: group, projects: [project_2.id] }) }
+
+      describe '#group_median' do
+        around do |example|
+          Timecop.freeze { example.run }
+        end
+
+        it 'counts median from issues with metrics' do
+          expect(stage.group_median).to eq(ISSUES_MEDIAN)
+        end
+      end
+
+      describe '#events' do
+        it 'exposes merge requests that close issues' do
+          result = stage.events
+
+          expect(result.count).to eq(1)
+          expect(result.map { |event| event[:title] }).to contain_exactly(issue_2_1.title)
+        end
+      end
+    end
+
     context 'when subgroup is given' do
       let(:subgroup) { create(:group, parent: group) }
       let(:project_4) { create(:project, group: subgroup) }