diff --git a/app/models/analytics/cycle_analytics/project_stage.rb b/app/models/analytics/cycle_analytics/project_stage.rb index a312bd24e78..23f0db0829b 100644 --- a/app/models/analytics/cycle_analytics/project_stage.rb +++ b/app/models/analytics/cycle_analytics/project_stage.rb @@ -9,6 +9,7 @@ module Analytics belongs_to :project alias_attribute :parent, :project + alias_attribute :parent_id, :project_id end end end diff --git a/app/models/concerns/analytics/cycle_analytics/stage.rb b/app/models/concerns/analytics/cycle_analytics/stage.rb index 0011ba10a9a..54e9a13d1ea 100644 --- a/app/models/concerns/analytics/cycle_analytics/stage.rb +++ b/app/models/concerns/analytics/cycle_analytics/stage.rb @@ -47,11 +47,17 @@ module Analytics !custom end - # The model that is going to be queried, Issue or MergeRequest - def subject_model + # The model class that is going to be queried, Issue or MergeRequest + def subject_class start_event.object_type end + def matches_with_stage_params?(stage_params) + default_stage? && + start_event_identifier.to_s.eql?(stage_params[:start_event_identifier].to_s) && + end_event_identifier.to_s.eql?(stage_params[:end_event_identifier].to_s) + end + private def validate_stage_event_pairs diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb index 7706119681d..ca50820a879 100644 --- a/app/models/merge_request_diff.rb +++ b/app/models/merge_request_diff.rb @@ -591,3 +591,5 @@ class MergeRequestDiff < ApplicationRecord end end end + +MergeRequestDiff.prepend_if_ee('EE::MergeRequestDiff') diff --git a/db/migrate/20190905074652_index_timestamp_columns_for_issue_metrics.rb b/db/migrate/20190905074652_index_timestamp_columns_for_issue_metrics.rb new file mode 100644 index 00000000000..e468b2decd9 --- /dev/null +++ b/db/migrate/20190905074652_index_timestamp_columns_for_issue_metrics.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +class IndexTimestampColumnsForIssueMetrics < ActiveRecord::Migration[5.2] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index(*index_arguments) + end + + def down + remove_concurrent_index(*index_arguments) + end + + private + + def index_arguments + [ + :issue_metrics, + [:issue_id, :first_mentioned_in_commit_at, :first_associated_with_milestone_at, :first_added_to_board_at], + { + name: 'index_issue_metrics_on_issue_id_and_timestamps' + } + ] + end +end diff --git a/db/migrate/20190919104119_index_timestamp_columns_for_merge_requests_creation_date.rb b/db/migrate/20190919104119_index_timestamp_columns_for_merge_requests_creation_date.rb new file mode 100644 index 00000000000..5e0d80630cd --- /dev/null +++ b/db/migrate/20190919104119_index_timestamp_columns_for_merge_requests_creation_date.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +class IndexTimestampColumnsForMergeRequestsCreationDate < ActiveRecord::Migration[5.2] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index(*index_arguments) + end + + def down + remove_concurrent_index(*index_arguments) + end + + private + + def index_arguments + [ + :merge_requests, + [:target_project_id, :created_at], + { + name: 'index_merge_requests_target_project_id_created_at' + } + ] + end +end diff --git a/db/schema.rb b/db/schema.rb index 352023ac40d..d1bd30a131e 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -1816,6 +1816,7 @@ ActiveRecord::Schema.define(version: 2019_10_04_134055) do t.datetime "first_added_to_board_at" t.datetime "created_at", null: false t.datetime "updated_at", null: false + t.index ["issue_id", "first_mentioned_in_commit_at", "first_associated_with_milestone_at", "first_added_to_board_at"], name: "index_issue_metrics_on_issue_id_and_timestamps" t.index ["issue_id"], name: "index_issue_metrics" end @@ -2226,6 +2227,7 @@ ActiveRecord::Schema.define(version: 2019_10_04_134055) do t.index ["source_project_id", "source_branch"], name: "index_merge_requests_on_source_project_id_and_source_branch" t.index ["state", "merge_status"], name: "index_merge_requests_on_state_and_merge_status", where: "(((state)::text = 'opened'::text) AND ((merge_status)::text = 'can_be_merged'::text))" t.index ["target_branch"], name: "index_merge_requests_on_target_branch" + t.index ["target_project_id", "created_at"], name: "index_merge_requests_target_project_id_created_at" t.index ["target_project_id", "iid"], name: "index_merge_requests_on_target_project_id_and_iid", unique: true t.index ["target_project_id", "iid"], name: "index_merge_requests_on_target_project_id_and_iid_opened", where: "((state)::text = 'opened'::text)" t.index ["target_project_id", "merge_commit_sha", "id"], name: "index_merge_requests_on_tp_id_and_merge_commit_sha_and_id" diff --git a/doc/user/project/merge_requests/browser_performance_testing.md b/doc/user/project/merge_requests/browser_performance_testing.md index 08feb970334..3cfc30a68e9 100644 --- a/doc/user/project/merge_requests/browser_performance_testing.md +++ b/doc/user/project/merge_requests/browser_performance_testing.md @@ -44,7 +44,7 @@ For instance, consider the following workflow: First of all, you need to define a job in your `.gitlab-ci.yml` file that generates the [Performance report artifact](../../../ci/yaml/README.md#artifactsreportsperformance-premium). For more information on how the Performance job should look like, check the -example on [Testing Browser Performance](../../../ci/examples/browser_performance.md). +example on [Configuring Browser Performance Testing](#configuring-browser-performance-testing). GitLab then checks this report, compares key performance metrics for each page between the source and target branches, and shows the information right on the merge request. @@ -60,11 +60,6 @@ report will be shown properly. ## Configuring Browser Performance Testing -NOTE: **Note:** -The job definition shown below is supported in GitLab 11.5 and later versions. -It also requires GitLab Runner 11.5 or later. For earlier versions, use the -[previous job definitions](#previous-job-definitions). - This example shows how to run the [sitespeed.io container](https://hub.docker.com/r/sitespeedio/sitespeed.io/) on your code by using GitLab CI/CD and [sitespeed.io](https://www.sitespeed.io) using Docker-in-Docker. @@ -73,29 +68,35 @@ First, you need GitLab Runner with [docker-in-docker build](../../../ci/docker/using_docker_build.md#use-docker-in-docker-workflow-with-docker-executor). Once you set up the Runner, add a new job to `.gitlab-ci.yml` that generates the -expected report: +expected report. + +For GitLab 12.4 and later, to define the `performance` job, you must +[include](../../../ci/yaml/README.md#includetemplate) the +[`Browser-Performance.gitlab-ci.yml` template](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml) +that's provided as a part of your GitLab installation. +For GitLab versions earlier than 12.4, you can copy and use the job as defined +in that template. + +CAUTION: **Caution:** +The job definition provided by the template does not support Kubernetes yet. For a complete example of a more complex setup +that works in Kubernetes, see [here](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml). + +Add the following to your `.gitlab-ci.yml` file: ```yaml +include: + template: Verify/Browser-Performance.gitlab-ci.yml + performance: - stage: performance - image: docker:git variables: URL: https://example.com - services: - - docker:stable-dind - script: - - mkdir gitlab-exporter - - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/master/index.js - - mkdir sitespeed-results - - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results $URL - - mv sitespeed-results/data/performance.json performance.json - artifacts: - paths: - - sitespeed-results/ - reports: - performance: performance.json ``` +CAUTION: **Caution:** +The job definition provided by the template is supported in GitLab 11.5 and later versions. +It also requires GitLab Runner 11.5 or later. For earlier versions, use the +[previous job definitions](#previous-job-definitions). + The above example will create a `performance` job in your CI/CD pipeline and will run sitespeed.io against the webpage you defined in `URL` to gather key metrics. The [GitLab plugin for sitespeed.io](https://gitlab.com/gitlab-org/gl-performance) @@ -106,6 +107,20 @@ take the latest Performance artifact available. The full HTML sitespeed.io report will also be saved as an artifact, and if you have [GitLab Pages](../pages/index.md) enabled, it can be viewed directly in your browser. +It is also possible to customize options by setting the `SITESPEED_OPTIONS` variable. +For example, this is how to override the number of runs sitespeed.io +will make on the given URL: + +```yaml +include: + template: Verify/Browser-Performance.gitlab-ci.yml + +performance: + variables: + URL: https://example.com + SITESPEED_OPTIONS: -n 5 +``` + For further customization options for sitespeed.io, including the ability to provide a list of URLs to test, please see the [Sitespeed.io Configuration](https://www.sitespeed.io/documentation/sitespeed.io/configuration/) documentation. @@ -126,8 +141,9 @@ set this up: as an artifact is as simple as `echo $CI_ENVIRONMENT_URL > environment_url.txt` in your job's `script`. 1. In the `performance` job, read the previous artifact into an environment - variable, like `$CI_ENVIRONMENT_URL`, and use it to parameterize the test - URLs. + variable, in this case `$URL` because this is what our sitespeed.io command + uses for the URL parameter. Because Review App URLs are dynamic, we define + the `URL` variable through `before_script` instead of `variables`. 1. You can now run the sitespeed.io container against the desired hostname and paths. @@ -138,6 +154,9 @@ stages: - deploy - performance +include: + template: Verify/Browser-Performance.gitlab-ci.yml + review: stage: deploy environment: @@ -155,28 +174,12 @@ review: - master performance: - stage: performance - image: docker:git - services: - - docker:stable-dind dependencies: - review - script: - - export CI_ENVIRONMENT_URL=$(cat environment_url.txt) - - mkdir gitlab-exporter - - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/master/index.js - - mkdir sitespeed-results - - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL" - - mv sitespeed-results/data/performance.json performance.json - artifacts: - paths: - - sitespeed-results/ - reports: - performance: performance.json + before_script: + - export URL=$(cat environment_url.txt) ``` -A complete example can be found in our [Auto DevOps CI YML](https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml). - ### Previous job definitions CAUTION: **Caution:** diff --git a/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb b/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb new file mode 100644 index 00000000000..33cbe1a62ef --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + class BaseQueryBuilder + include Gitlab::CycleAnalytics::MetricsTables + + delegate :subject_class, to: :stage + + # rubocop: disable CodeReuse/ActiveRecord + + def initialize(stage:, params: {}) + @stage = stage + @params = params + end + + def build + query = subject_class + query = filter_by_parent_model(query) + query = filter_by_time_range(query) + query = stage.start_event.apply_query_customization(query) + query = stage.end_event.apply_query_customization(query) + query.where(duration_condition) + end + + private + + attr_reader :stage, :params + + def duration_condition + stage.end_event.timestamp_projection.gteq(stage.start_event.timestamp_projection) + end + + def filter_by_parent_model(query) + if parent_class.eql?(Project) + if subject_class.eql?(Issue) + query.where(project_id: stage.parent_id) + elsif subject_class.eql?(MergeRequest) + query.where(target_project_id: stage.parent_id) + else + raise ArgumentError, "unknown subject_class: #{subject_class}" + end + else + raise ArgumentError, "unknown parent_class: #{parent_class}" + end + end + + def filter_by_time_range(query) + from = params.fetch(:from, 30.days.ago) + to = params[:to] + + query = query.where(subject_table[:created_at].gteq(from)) + query = query.where(subject_table[:created_at].lteq(to)) if to + query + end + + def subject_table + subject_class.arel_table + end + + def parent_class + stage.parent.class + end + + # rubocop: enable CodeReuse/ActiveRecord + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/data_collector.rb b/lib/gitlab/analytics/cycle_analytics/data_collector.rb new file mode 100644 index 00000000000..0c0f737f2c9 --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/data_collector.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + # Arguments: + # stage - an instance of CycleAnalytics::ProjectStage or CycleAnalytics::GroupStage + # params: + # current_user: an instance of User + # from: DateTime + # to: DateTime + class DataCollector + include Gitlab::Utils::StrongMemoize + + def initialize(stage:, params: {}) + @stage = stage + @params = params + end + + def records_fetcher + strong_memoize(:records_fetcher) do + RecordsFetcher.new(stage: stage, query: query, params: params) + end + end + + def median + strong_memoize(:median) do + Median.new(stage: stage, query: query) + end + end + + private + + attr_reader :stage, :params + + def query + BaseQueryBuilder.new(stage: stage, params: params).build + end + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/default_stages.rb b/lib/gitlab/analytics/cycle_analytics/default_stages.rb index 711645800fb..8e70236ce75 100644 --- a/lib/gitlab/analytics/cycle_analytics/default_stages.rb +++ b/lib/gitlab/analytics/cycle_analytics/default_stages.rb @@ -92,8 +92,8 @@ module Gitlab name: 'production', custom: false, relative_position: 7, - start_event_identifier: :merge_request_merged, - end_event_identifier: :merge_request_first_deployed_to_production + start_event_identifier: :issue_created, + end_event_identifier: :production_stage_end } end end diff --git a/lib/gitlab/analytics/cycle_analytics/median.rb b/lib/gitlab/analytics/cycle_analytics/median.rb new file mode 100644 index 00000000000..41883a80338 --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/median.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + class Median + include StageQueryHelpers + + def initialize(stage:, query:) + @stage = stage + @query = query + end + + def seconds + @query = @query.select(median_duration_in_seconds.as('median')) + result = execute_query(@query).first || {} + + result['median'] ? result['median'].to_i : nil + end + + private + + attr_reader :stage + + def percentile_cont + percentile_cont_ordering = Arel::Nodes::UnaryOperation.new(Arel::Nodes::SqlLiteral.new('ORDER BY'), duration) + Arel::Nodes::NamedFunction.new( + 'percentile_cont(0.5) WITHIN GROUP', + [percentile_cont_ordering] + ) + end + + def median_duration_in_seconds + Arel::Nodes::Extract.new(percentile_cont, :epoch) + end + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb new file mode 100644 index 00000000000..90d03142b2a --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb @@ -0,0 +1,132 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + class RecordsFetcher + include Gitlab::Utils::StrongMemoize + include StageQueryHelpers + include Gitlab::CycleAnalytics::MetricsTables + + MAX_RECORDS = 20 + + MAPPINGS = { + Issue => { + finder_class: IssuesFinder, + serializer_class: AnalyticsIssueSerializer, + includes_for_query: { project: [:namespace], author: [] }, + columns_for_select: %I[title iid id created_at author_id project_id] + }, + MergeRequest => { + finder_class: MergeRequestsFinder, + serializer_class: AnalyticsMergeRequestSerializer, + includes_for_query: { target_project: [:namespace], author: [] }, + columns_for_select: %I[title iid id created_at author_id state target_project_id] + } + }.freeze + + delegate :subject_class, to: :stage + + def initialize(stage:, query:, params: {}) + @stage = stage + @query = query + @params = params + end + + def serialized_records + strong_memoize(:serialized_records) do + # special case (legacy): 'Test' and 'Staging' stages should show Ci::Build records + if default_test_stage? || default_staging_stage? + AnalyticsBuildSerializer.new.represent(ci_build_records.map { |e| e['build'] }) + else + records.map do |record| + project = record.project + attributes = record.attributes.merge({ + project_path: project.path, + namespace_path: project.namespace.path, + author: record.author + }) + serializer.represent(attributes) + end + end + end + end + + private + + attr_reader :stage, :query, :params + + def finder_query + MAPPINGS + .fetch(subject_class) + .fetch(:finder_class) + .new(params.fetch(:current_user), finder_params.fetch(stage.parent.class)) + .execute + end + + def columns + MAPPINGS.fetch(subject_class).fetch(:columns_for_select).map do |column_name| + subject_class.arel_table[column_name] + end + end + + # EE will override this to include Group rules + def finder_params + { + Project => { project_id: stage.parent_id } + } + end + + def default_test_stage? + stage.matches_with_stage_params?(Gitlab::Analytics::CycleAnalytics::DefaultStages.params_for_test_stage) + end + + def default_staging_stage? + stage.matches_with_stage_params?(Gitlab::Analytics::CycleAnalytics::DefaultStages.params_for_staging_stage) + end + + def serializer + MAPPINGS.fetch(subject_class).fetch(:serializer_class).new + end + + # Loading Ci::Build records instead of MergeRequest records + # rubocop: disable CodeReuse/ActiveRecord + def ci_build_records + ci_build_join = mr_metrics_table + .join(build_table) + .on(mr_metrics_table[:pipeline_id].eq(build_table[:commit_id])) + .join_sources + + q = ordered_and_limited_query + .joins(ci_build_join) + .select(build_table[:id], round_duration_to_seconds.as('total_time')) + + results = execute_query(q).to_a + + Gitlab::CycleAnalytics::Updater.update!(results, from: 'id', to: 'build', klass: ::Ci::Build.includes({ project: [:namespace], user: [], pipeline: [] })) + end + + def ordered_and_limited_query + query + .reorder(stage.end_event.timestamp_projection.desc) + .limit(MAX_RECORDS) + end + + def records + results = finder_query + .merge(ordered_and_limited_query) + .select(*columns, round_duration_to_seconds.as('total_time')) + + # using preloader instead of includes to avoid AR generating a large column list + ActiveRecord::Associations::Preloader.new.preload( + results, + MAPPINGS.fetch(subject_class).fetch(:includes_for_query) + ) + + results + end + # rubocop: enable CodeReuse/ActiveRecord + end + end + end +end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb b/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb new file mode 100644 index 00000000000..34c726b2254 --- /dev/null +++ b/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + module CycleAnalytics + module StageQueryHelpers + def execute_query(query) + ActiveRecord::Base.connection.execute(query.to_sql) + end + + def zero_interval + Arel::Nodes::NamedFunction.new("CAST", [Arel.sql("'0' AS INTERVAL")]) + end + + def round_duration_to_seconds + Arel::Nodes::Extract.new(duration, :epoch) + end + + def duration + Arel::Nodes::Subtraction.new( + stage.end_event.timestamp_projection, + stage.start_event.timestamp_projection + ) + end + end + end + end +end diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml new file mode 100644 index 00000000000..eced181e966 --- /dev/null +++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml @@ -0,0 +1,29 @@ +# Read more about the feature here: https://docs.gitlab.com/ee/user/project/merge_requests/browser_performance_testing.html + +stages: + - build + - test + - deploy + - performance + +performance: + stage: performance + image: docker:git + variables: + URL: https://example.com + SITESPEED_VERSION: 6.3.1 + SITESPEED_OPTIONS: '' + services: + - docker:stable-dind + script: + - mkdir gitlab-exporter + - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/master/index.js + - mkdir sitespeed-results + - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --outputFolder sitespeed-results $URL $SITESPEED_OPTIONS + - mv sitespeed-results/data/performance.json performance.json + artifacts: + paths: + - performance.json + - sitespeed-results/ + reports: + performance: performance.json diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 8ec7a725871..c10d94d96ab 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -6030,6 +6030,15 @@ msgstr "" msgid "Environments|An error occurred while fetching the environments." msgstr "" +msgid "Environments|An error occurred while fetching the logs" +msgstr "" + +msgid "Environments|An error occurred while fetching the logs - Error: %{message}" +msgstr "" + +msgid "Environments|An error occurred while fetching the logs for this environment or pod. Please try again" +msgstr "" + msgid "Environments|An error occurred while making the request." msgstr "" @@ -6075,9 +6084,6 @@ msgstr "" msgid "Environments|No deployments yet" msgstr "" -msgid "Environments|No pod name has been specified" -msgstr "" - msgid "Environments|Note that this action will stop the environment, but it will %{emphasisStart}not%{emphasisEnd} have an effect on any existing deployment due to no “stop environment action” being defined in the %{ciConfigLinkStart}.gitlab-ci.yml%{ciConfigLinkEnd} file." msgstr "" @@ -14986,9 +14992,6 @@ msgstr "" msgid "Something went wrong on our end." msgstr "" -msgid "Something went wrong on our end. %{message}" -msgstr "" - msgid "Something went wrong on our end. Please try again!" msgstr "" diff --git a/spec/factories/analytics/cycle_analytics/project_stages.rb b/spec/factories/analytics/cycle_analytics/project_stages.rb new file mode 100644 index 00000000000..6f8c140ed8a --- /dev/null +++ b/spec/factories/analytics/cycle_analytics/project_stages.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :cycle_analytics_project_stage, class: Analytics::CycleAnalytics::ProjectStage do + project + sequence(:name) { |n| "Stage ##{n}" } + hidden { false } + issue_stage + + trait :issue_stage do + start_event_identifier { Gitlab::Analytics::CycleAnalytics::StageEvents::IssueCreated.identifier } + end_event_identifier { Gitlab::Analytics::CycleAnalytics::StageEvents::IssueStageEnd.identifier } + end + end +end diff --git a/spec/frontend/fixtures/static/environments_logs.html b/spec/frontend/fixtures/static/environments_logs.html index 6179d3dbc23..ccf9c364154 100644 --- a/spec/frontend/fixtures/static/environments_logs.html +++ b/spec/frontend/fixtures/static/environments_logs.html @@ -1,29 +1,102 @@ -
-
+
+
-
- -
- -
+
+
+
+ +
+ +
+
+
+
+ +
+ +
+
+
-
-
- +
+
+
-
- +
+
-
-
- +
+
+
-
+ 
+        
+        

+          

+          

+          

+        

+
diff --git a/spec/lib/gitlab/analytics/cycle_analytics/base_query_builder_spec.rb b/spec/lib/gitlab/analytics/cycle_analytics/base_query_builder_spec.rb new file mode 100644 index 00000000000..0fc9d3c1e9e --- /dev/null +++ b/spec/lib/gitlab/analytics/cycle_analytics/base_query_builder_spec.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::Analytics::CycleAnalytics::BaseQueryBuilder do + let_it_be(:project) { create(:project, :empty_repo) } + let_it_be(:mr1) { create(:merge_request, target_project: project, source_project: project, allow_broken: true, created_at: 3.months.ago) } + let_it_be(:mr2) { create(:merge_request, target_project: project, source_project: project, allow_broken: true, created_at: 1.month.ago) } + let(:params) { {} } + let(:records) do + stage = build(:cycle_analytics_project_stage, { + start_event_identifier: :merge_request_created, + end_event_identifier: :merge_request_merged, + project: project + }) + described_class.new(stage: stage, params: params).build.to_a + end + + before do + mr1.metrics.update!(merged_at: 1.month.ago) + mr2.metrics.update!(merged_at: Time.now) + end + + around do |example| + Timecop.freeze { example.run } + end + + describe 'date range parameters' do + context 'when filters by only the `from` parameter' do + before do + params[:from] = 4.months.ago + end + + it { expect(records.size).to eq(2) } + end + + context 'when filters by both `from` and `to` parameters' do + before do + params.merge!(from: 4.months.ago, to: 2.months.ago) + end + + it { expect(records.size).to eq(1) } + end + + context 'invalid date range is provided' do + before do + params.merge!(from: 1.month.ago, to: 10.months.ago) + end + + it { expect(records.size).to eq(0) } + end + end + + it 'scopes query within the target project' do + other_mr = create(:merge_request, source_project: create(:project), allow_broken: true, created_at: 2.months.ago) + other_mr.metrics.update!(merged_at: 1.month.ago) + + params[:from] = 1.year.ago + + expect(records.size).to eq(2) + end +end diff --git a/spec/lib/gitlab/analytics/cycle_analytics/records_fetcher_spec.rb b/spec/lib/gitlab/analytics/cycle_analytics/records_fetcher_spec.rb new file mode 100644 index 00000000000..334cab0b799 --- /dev/null +++ b/spec/lib/gitlab/analytics/cycle_analytics/records_fetcher_spec.rb @@ -0,0 +1,131 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::Analytics::CycleAnalytics::RecordsFetcher do + around do |example| + Timecop.freeze { example.run } + end + + let_it_be(:project) { create(:project, :empty_repo) } + let_it_be(:user) { create(:user) } + + subject do + Gitlab::Analytics::CycleAnalytics::DataCollector.new( + stage: stage, + params: { + from: 1.year.ago, + current_user: user + } + ).records_fetcher.serialized_records + end + + describe '#serialized_records' do + shared_context 'when records are loaded by maintainer' do + before do + project.add_user(user, Gitlab::Access::MAINTAINER) + end + + it 'returns all records' do + expect(subject.size).to eq(2) + end + end + + describe 'for issue based stage' do + let_it_be(:issue1) { create(:issue, project: project) } + let_it_be(:issue2) { create(:issue, project: project, confidential: true) } + let(:stage) do + build(:cycle_analytics_project_stage, { + start_event_identifier: :plan_stage_start, + end_event_identifier: :issue_first_mentioned_in_commit, + project: project + }) + end + + before do + issue1.metrics.update(first_added_to_board_at: 3.days.ago, first_mentioned_in_commit_at: 2.days.ago) + issue2.metrics.update(first_added_to_board_at: 3.days.ago, first_mentioned_in_commit_at: 2.days.ago) + end + + context 'when records are loaded by guest' do + before do + project.add_user(user, Gitlab::Access::GUEST) + end + + it 'filters out confidential issues' do + expect(subject.size).to eq(1) + expect(subject.first[:iid].to_s).to eq(issue1.iid.to_s) + end + end + + include_context 'when records are loaded by maintainer' + end + + describe 'for merge request based stage' do + let(:mr1) { create(:merge_request, created_at: 5.days.ago, source_project: project, allow_broken: true) } + let(:mr2) { create(:merge_request, created_at: 4.days.ago, source_project: project, allow_broken: true) } + let(:stage) do + build(:cycle_analytics_project_stage, { + start_event_identifier: :merge_request_created, + end_event_identifier: :merge_request_merged, + project: project + }) + end + + before do + mr1.metrics.update(merged_at: 3.days.ago) + mr2.metrics.update(merged_at: 3.days.ago) + end + + include_context 'when records are loaded by maintainer' + end + + describe 'special case' do + let(:mr1) { create(:merge_request, source_project: project, allow_broken: true, created_at: 20.days.ago) } + let(:mr2) { create(:merge_request, source_project: project, allow_broken: true, created_at: 19.days.ago) } + let(:ci_build1) { create(:ci_build) } + let(:ci_build2) { create(:ci_build) } + let(:default_stages) { Gitlab::Analytics::CycleAnalytics::DefaultStages } + let(:stage) { build(:cycle_analytics_project_stage, default_stages.params_for_test_stage.merge(project: project)) } + + before do + mr1.metrics.update!({ + merged_at: 5.days.ago, + first_deployed_to_production_at: 1.day.ago, + latest_build_started_at: 5.days.ago, + latest_build_finished_at: 1.day.ago, + pipeline: ci_build1.pipeline + }) + mr2.metrics.update!({ + merged_at: 10.days.ago, + first_deployed_to_production_at: 5.days.ago, + latest_build_started_at: 9.days.ago, + latest_build_finished_at: 7.days.ago, + pipeline: ci_build2.pipeline + }) + end + + context 'returns build records' do + shared_examples 'orders build records by `latest_build_finished_at`' do + it 'orders by `latest_build_finished_at`' do + build_ids = subject.map { |item| item[:id] } + + expect(build_ids).to eq([ci_build1.id, ci_build2.id]) + end + end + + context 'when requesting records for default test stage' do + include_examples 'orders build records by `latest_build_finished_at`' + end + + context 'when requesting records for default staging stage' do + before do + stage.assign_attributes(default_stages.params_for_staging_stage) + end + + include_examples 'orders build records by `latest_build_finished_at`' + end + end + end + end +end diff --git a/spec/lib/gitlab/cycle_analytics/code_stage_spec.rb b/spec/lib/gitlab/cycle_analytics/code_stage_spec.rb index 6959778661b..aa12bc21d22 100644 --- a/spec/lib/gitlab/cycle_analytics/code_stage_spec.rb +++ b/spec/lib/gitlab/cycle_analytics/code_stage_spec.rb @@ -26,6 +26,13 @@ describe Gitlab::CycleAnalytics::CodeStage do it_behaves_like 'base stage' + context 'when using the new query backend' do + include_examples 'Gitlab::Analytics::CycleAnalytics::DataCollector backend examples' do + let(:expected_record_count) { 2 } + let(:expected_ordered_attribute_values) { [mr_2.title, mr_1.title] } + end + end + describe '#project_median' do around do |example| Timecop.freeze { example.run } diff --git a/spec/lib/gitlab/cycle_analytics/issue_stage_spec.rb b/spec/lib/gitlab/cycle_analytics/issue_stage_spec.rb index b99debf8cac..497db88d850 100644 --- a/spec/lib/gitlab/cycle_analytics/issue_stage_spec.rb +++ b/spec/lib/gitlab/cycle_analytics/issue_stage_spec.rb @@ -21,6 +21,13 @@ describe Gitlab::CycleAnalytics::IssueStage do it_behaves_like 'base stage' + context 'when using the new query backend' do + include_examples 'Gitlab::Analytics::CycleAnalytics::DataCollector backend examples' do + let(:expected_record_count) { 3 } + let(:expected_ordered_attribute_values) { [issue_3.title, issue_2.title, issue_1.title] } + end + end + describe '#median' do around do |example| Timecop.freeze { example.run } diff --git a/spec/lib/gitlab/cycle_analytics/plan_stage_spec.rb b/spec/lib/gitlab/cycle_analytics/plan_stage_spec.rb index 60d180015c4..01a46f5ba65 100644 --- a/spec/lib/gitlab/cycle_analytics/plan_stage_spec.rb +++ b/spec/lib/gitlab/cycle_analytics/plan_stage_spec.rb @@ -21,6 +21,13 @@ describe Gitlab::CycleAnalytics::PlanStage do it_behaves_like 'base stage' + context 'when using the new query backend' do + include_examples 'Gitlab::Analytics::CycleAnalytics::DataCollector backend examples' do + let(:expected_record_count) { 2 } + let(:expected_ordered_attribute_values) { [issue_1.title, issue_2.title] } + end + end + describe '#project_median' do around do |example| Timecop.freeze { example.run } diff --git a/spec/lib/gitlab/cycle_analytics/shared_stage_spec.rb b/spec/lib/gitlab/cycle_analytics/shared_stage_spec.rb index b3a6c2c4239..c5b17aafdd2 100644 --- a/spec/lib/gitlab/cycle_analytics/shared_stage_spec.rb +++ b/spec/lib/gitlab/cycle_analytics/shared_stage_spec.rb @@ -52,3 +52,21 @@ shared_examples 'calculate #median with date range' do it { expect(stage.project_median).to eq(nil) } end end + +shared_examples 'Gitlab::Analytics::CycleAnalytics::DataCollector backend examples' do + let(:stage_params) { Gitlab::Analytics::CycleAnalytics::DefaultStages.send("params_for_#{stage_name}_stage").merge(project: project) } + let(:stage) { Analytics::CycleAnalytics::ProjectStage.new(stage_params) } + let(:data_collector) { Gitlab::Analytics::CycleAnalytics::DataCollector.new(stage: stage, params: { from: stage_options[:from], current_user: project.creator }) } + let(:attribute_to_verify) { :title } + + context 'provides the same results as the old implementation' do + it 'for the median' do + expect(data_collector.median.seconds).to eq(ISSUES_MEDIAN) + end + + it 'for the list of event records' do + records = data_collector.records_fetcher.serialized_records + expect(records.map { |event| event[attribute_to_verify] }).to eq(expected_ordered_attribute_values) + end + end +end diff --git a/spec/lib/gitlab/cycle_analytics/test_stage_spec.rb b/spec/lib/gitlab/cycle_analytics/test_stage_spec.rb index bbb53cefae7..e347f36dfce 100644 --- a/spec/lib/gitlab/cycle_analytics/test_stage_spec.rb +++ b/spec/lib/gitlab/cycle_analytics/test_stage_spec.rb @@ -12,17 +12,20 @@ describe Gitlab::CycleAnalytics::TestStage do it_behaves_like 'base stage' describe '#median' do + let(:mr_1) { create(:merge_request, :closed, source_project: project, created_at: 60.minutes.ago) } + let(:mr_2) { create(:merge_request, :closed, source_project: project, created_at: 40.minutes.ago, source_branch: 'A') } + let(:mr_3) { create(:merge_request, source_project: project, created_at: 10.minutes.ago, source_branch: 'B') } + let(:mr_4) { create(:merge_request, source_project: project, created_at: 10.minutes.ago, source_branch: 'C') } + let(:mr_5) { create(:merge_request, source_project: project, created_at: 10.minutes.ago, source_branch: 'D') } + let(:ci_build1) { create(:ci_build, project: project) } + let(:ci_build2) { create(:ci_build, project: project) } + before do issue_1 = create(:issue, project: project, created_at: 90.minutes.ago) issue_2 = create(:issue, project: project, created_at: 60.minutes.ago) issue_3 = create(:issue, project: project, created_at: 60.minutes.ago) - mr_1 = create(:merge_request, :closed, source_project: project, created_at: 60.minutes.ago) - mr_2 = create(:merge_request, :closed, source_project: project, created_at: 40.minutes.ago, source_branch: 'A') - mr_3 = create(:merge_request, source_project: project, created_at: 10.minutes.ago, source_branch: 'B') - mr_4 = create(:merge_request, source_project: project, created_at: 10.minutes.ago, source_branch: 'C') - mr_5 = create(:merge_request, source_project: project, created_at: 10.minutes.ago, source_branch: 'D') - mr_1.metrics.update!(latest_build_started_at: 32.minutes.ago, latest_build_finished_at: 2.minutes.ago) - mr_2.metrics.update!(latest_build_started_at: 62.minutes.ago, latest_build_finished_at: 32.minutes.ago) + mr_1.metrics.update!(latest_build_started_at: 32.minutes.ago, latest_build_finished_at: 2.minutes.ago, pipeline_id: ci_build1.commit_id) + mr_2.metrics.update!(latest_build_started_at: 62.minutes.ago, latest_build_finished_at: 32.minutes.ago, pipeline_id: ci_build2.commit_id) mr_3.metrics.update!(latest_build_started_at: nil, latest_build_finished_at: nil) mr_4.metrics.update!(latest_build_started_at: nil, latest_build_finished_at: nil) mr_5.metrics.update!(latest_build_started_at: nil, latest_build_finished_at: nil) @@ -43,5 +46,13 @@ describe Gitlab::CycleAnalytics::TestStage do end include_examples 'calculate #median with date range' + + context 'when using the new query backend' do + include_examples 'Gitlab::Analytics::CycleAnalytics::DataCollector backend examples' do + let(:expected_record_count) { 2 } + let(:attribute_to_verify) { :id } + let(:expected_ordered_attribute_values) { [mr_1.metrics.pipeline.builds.first.id, mr_2.metrics.pipeline.builds.first.id] } + end + end end end diff --git a/spec/support/shared_examples/cycle_analytics_stage_examples.rb b/spec/support/shared_examples/cycle_analytics_stage_shared_examples.rb similarity index 75% rename from spec/support/shared_examples/cycle_analytics_stage_examples.rb rename to spec/support/shared_examples/cycle_analytics_stage_shared_examples.rb index dc2ea229171..afa035d039a 100644 --- a/spec/support/shared_examples/cycle_analytics_stage_examples.rb +++ b/spec/support/shared_examples/cycle_analytics_stage_shared_examples.rb @@ -55,11 +55,11 @@ shared_examples_for 'cycle analytics stage' do end end - describe '#subject_model' do + describe '#subject_class' do it 'infers the model from the start event' do stage = described_class.new(valid_params) - expect(stage.subject_model).to eq(MergeRequest) + expect(stage.subject_class).to eq(MergeRequest) end end @@ -78,4 +78,30 @@ shared_examples_for 'cycle analytics stage' do expect(stage.end_event).to be_a_kind_of(Gitlab::Analytics::CycleAnalytics::StageEvents::MergeRequestMerged) end end + + describe '#matches_with_stage_params?' do + let(:params) { Gitlab::Analytics::CycleAnalytics::DefaultStages.params_for_test_stage } + + it 'matches with default stage params' do + stage = described_class.new(params) + + expect(stage).to be_default_stage + expect(stage).to be_matches_with_stage_params(params) + end + + it "mismatches when the stage is custom" do + stage = described_class.new(params.merge(custom: true)) + + expect(stage).not_to be_default_stage + expect(stage).not_to be_matches_with_stage_params(params) + end + end + + describe '#parent_id' do + it "delegates to 'parent_name'_id attribute" do + stage = described_class.new(parent: parent) + + expect(stage.parent_id).to eq(parent.id) + end + end end