Add latest changes from gitlab-org/gitlab@master

.gitlab/ci/global.gitlab-ci.yml

@@ -13,6 +13,8 @@
 .default-before_script:
   before_script:
     - date
+    - export GOPATH=$CI_PROJECT_DIR/.go
+    - mkdir -p $GOPATH
     - source scripts/utils.sh
     - source scripts/prepare_build.sh
     - date
@@ -22,6 +24,7 @@
   cache:
     key: "debian-stretch-ruby-2.6.3-node-12.x"
     paths:
+      - .go/pkg/mod
       - vendor/ruby
       - .yarn-cache/
       - vendor/gitaly-ruby
@@ -52,7 +55,7 @@
       - "{babel.config,jest.config}.js"
       - "config.ru"
       - "{package.json,yarn.lock}"
-      - "{app,bin,config,danger,db,ee,fixtures,haml_lint,lib,public,rubocop,scripts,spec,symbol,vendor}/**/*"
+      - "{app,bin,config,danger,db,ee,fixtures,haml_lint,lib,locale,public,rubocop,scripts,spec,symbol,vendor}/**/*"
       - "doc/README.md"  # Some RSpec test rely on this file
 
 .only-qa-changes:
@@ -83,7 +86,7 @@
       - "{babel.config,jest.config}.js"
       - "config.ru"
       - "{package.json,yarn.lock}"
-      - "{app,bin,config,danger,db,ee,fixtures,haml_lint,lib,public,rubocop,scripts,spec,symbol,vendor}/**/*"
+      - "{app,bin,config,danger,db,ee,fixtures,haml_lint,lib,locale,public,rubocop,scripts,spec,symbol,vendor}/**/*"
       - "doc/README.md"  # Some RSpec test rely on this file
       - ".dockerignore"
       - "qa/**/*"
@@ -123,3 +126,4 @@
   only:
     variables:
       - $CI_PROJECT_NAME == "gitlab"
+      - $CI_PROJECT_NAME == "gitlab-ee"  # Support former project name for forks/mirrors

.gitlab/ci/rails.gitlab-ci.yml

@@ -95,6 +95,8 @@ setup-test-env:
       - tmp/tests
       - config/secrets.yml
       - vendor/gitaly-ruby
+  cache:
+    policy: pull-push
 
 rspec unit pg:
   extends: .rspec-base-pg

.gitlab/ci/setup.gitlab-ci.yml

@@ -43,5 +43,6 @@ no_ee_check:
     - scripts/no-ee-check
   only:
     variables:
-      - $CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_NAME == "gitlab-foss"
-      - $CI_SERVER_HOST == "dev.gitlab.org" && $CI_PROJECT_NAME == "gitlabhq"
+      - $CI_PROJECT_NAME == "gitlab-foss"
+      - $CI_PROJECT_NAME == "gitlab-ce"  # Support former project name for forks/mirrors
+      - $CI_PROJECT_NAME == "gitlabhq"  # Support former project name for dev

CHANGELOG-EE.md

@@ -1,5 +1,9 @@
 Please view this file on the master branch, on stable branches it's out of date.
 
+## 12.3.1
+
+- No changes.
+
 ## 12.3.0
 
 ### Security (3 changes)

app/models/ci/pipeline.rb

@@ -771,6 +771,18 @@ module Ci
       end
     end
 
+    def all_worktree_paths
+      strong_memoize(:all_worktree_paths) do
+        project.repository.ls_files(sha)
+      end
+    end
+
+    def top_level_worktree_paths
+      strong_memoize(:top_level_worktree_paths) do
+        project.repository.tree(sha).blobs.map(&:path)
+      end
+    end
+
     def default_branch?
       ref == project.default_branch
     end

app/views/dashboard/projects/_zero_authorized_projects.html.haml

@@ -2,11 +2,11 @@
   .section-container.section-welcome{ class: "#{ 'section-admin-welcome' if current_user.admin? }" }
     .container.section-body
       .row
-        .blank-state-welcome
+        .blank-state-welcome.w-100
           %h2.blank-state-welcome-title
-            Welcome to GitLab
+            = _('Welcome to GitLab')
           %p.blank-state-text
-            Code, test, and deploy together
+            = _('Faster releases. Better code. Less pain.')
         .blank-state-row
           %div{ class: ('column-large' if has_start_trial?) }
             - if current_user.admin?

changelogs/unreleased/32457-change-welcome-to-gitlab-message.yml

@@ -0,0 +1,5 @@
+---
+title: Change welcome message and make translatable
+merge_request: 17391
+author:
+type: other

changelogs/unreleased/job_file_matching.yml

@@ -0,0 +1,5 @@
+---
+title: Add file matching rule to flexible CI rules
+merge_request: 16574
+author:
+type: added

doc/administration/gitaly/index.md

@@ -564,6 +564,109 @@ concurrency limiter, not a rate limiter. If a client makes 1000 requests
 in a row in a very short timespan, the concurrency will not exceed 1,
 and this mechanism (the concurrency limiter) will do nothing.
 
+## Rotating a Gitaly authentication token
+
+Rotating credentials in a production environment often either requires
+downtime, or causes outages, or both. If you are careful, though, you
+*can* rotate Gitaly credentials without a service interruption.
+
+This procedure also works if you are running GitLab on a single server.
+In that case, "Gitaly servers" and "Gitaly clients" refers to the same
+machine.
+
+### 1. Monitor current authentication behavior
+
+Use prometheus to see what the current authentication behavior of your
+GitLab installation is.
+
+```
+sum(rate(gitaly_authentications_total[5m])) by (enforced, status)
+```
+
+In a system where authentication is configured correctly, and where you
+have live traffic, you will see something like this:
+
+```
+{enforced="true",status="ok"}  4424.985419441742
+```
+
+There may also be other numbers with rate 0. We only care about the
+non-zero numbers.
+
+The only non-zero number should have `enforced="true",status="ok"`. If
+you have other non-zero numbers, something is wrong in your
+configuration.
+
+The 'status="ok"' number reflects your current request rate. In the example
+above, Gitaly is handling about 4000 requests per second.
+
+Now you have established that you can monitor the Gitaly authentication
+behavior of your GitLab installation.
+
+### 2. Reconfigure all Gitaly servers to be in "auth transitioning" mode
+
+The second step is to temporarily disable authentication on the Gitaly servers.
+
+```ruby
+# in /etc/gitlab/gitlab.rb
+gitaly['auth_transitioning'] = true
+```
+
+After you have applied this, your prometheus query should return
+something like this:
+
+```
+{enforced="false",status="would be ok"}  4424.985419441742
+```
+
+Because `enforced="false"`, it will be safe to start rolling out the new
+token.
+
+### 3. Update Gitaly token on all clients and servers
+
+```ruby
+# in /etc/gitlab/gitlab.rb
+
+gitaly['auth_token'] = 'my new secret token'
+```
+
+Remember to apply this on both your Gitaly clients *and* servers. If you
+check your prometheus query while this change is being rolled out, you
+will see non-zero values for the `enforced="false",status="denied"` counter.
+
+### 4. Use prometheus to ensure there are no authentication failures
+
+After you applied the Gitaly token change everywhere, and all services
+involved have been restarted, you should will temporarily see a mix of
+`status="would be ok"` and `status="denied"`.
+
+After the new token has been picked up by all Gitaly clients and
+servers, the **only non-zero rate** should be
+`enforced="false",status="would be ok"`.
+
+### 5. Disable "auth transitioning" Mode
+
+Now we turn off the 'auth transitioning' mode. These final steps are
+important: without them, you have **no authentication**.
+
+Update the configuration on your Gitaly servers:
+
+```ruby
+# in /etc/gitlab/gitlab.rb
+gitaly['auth_transitioning'] = false
+```
+
+### 6. Verify that authentication is enforced again
+
+Refresh your prometheus query. You should now see the same kind of
+result as you did in the beginning:
+
+```
+{enforced="true",status="ok"}  4424.985419441742
+```
+
+Note that `enforced="true"`, meaning that authentication is being enforced.
+
 ## Troubleshooting Gitaly
 
 ### `gitaly-debug`

doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md

@@ -715,13 +715,15 @@ Namespace.find_by_full_path("user/proj").namespace_statistics.update(shared_runn
 project = Project.find_by_full_path('')
 builds_with_artifacts =  project.builds.with_artifacts_archive
 
-# Prior to 10.6 the above line would be:
-# builds_with_artifacts = project.builds.with_artifacts
-
 # Instance-wide:
-builds_with_artifacts = Ci::Build.with_artifacts
+builds_with_artifacts = Ci::Build.with_artifacts_archive
+
+# Prior to 10.6 the above lines would be:
+# builds_with_artifacts =  project.builds.with_artifacts
+# builds_with_artifacts = Ci::Build.with_artifacts
 
 ### CLEAR THEM OUT
+# Note that this will also erase artifacts that developers marked to "Keep"
 builds_to_clear = builds_with_artifacts.where("finished_at < ?", 1.week.ago)
 builds_to_clear.each do |build|
   build.artifacts_expire_at = Time.now

doc/ci/multi_project_pipelines.md

@@ -116,7 +116,8 @@ staging:
 ```
 
 Use a `project` keyword to specify full path to a downstream project. Use
-a `branch` keyword to specify a branch name.
+a `branch` keyword to specify a branch name. Variable expansion is supported in
+the `branch` property.
 
 GitLab will use a commit that is currently on the HEAD of the branch when
 creating a downstream pipeline.

doc/development/performance.md

@@ -251,6 +251,36 @@ These results can also be placed into a PostgreSQL database by setting the
 `RSPEC_PROFILING_POSTGRES_URL` variable. This is used to profile the test suite
 when running in the CI environment.
 
+## Memory profiling
+
+One of the reasons of the increased memory footprint could be Ruby memory fragmentation.
+
+To diagnose it, you can visualize Ruby heap as described in [this post by Aaron Patterson](https://tenderlovemaking.com/2017/09/27/visualizing-your-ruby-heap.html).
+
+To start, you want to dump the heap of the process you are investigating to a JSON file.  
+
+You need to run the command inside the process you are exploring, you may do that with `rbtrace`.  
+`rbtrace` is already present in GitLab `Gemfile`, you just need to require it.  
+It could be achieved running webserver or Sidekiq with the environment variable set to `ENABLE_RBTRACE=1`.
+
+To get the heap dump:
+
+```ruby
+bundle exec rbtrace -p <PID> -e 'File.open("heap.json", "wb") { |t| ObjectSpace.dump_all(output: t) }'
+```
+
+Having the JSON, you finally could render a picture using the script [provided by Aaron](https://gist.github.com/tenderlove/f28373d56fdd03d8b514af7191611b88) or similar:
+
+```sh
+ruby heapviz.rb heap.json
+```
+  
+Fragmented Ruby heap snapshot could look like this:
+
+![Ruby heap fragmentation](img/memory_ruby_heap_fragmentation.png)
+
+Memory fragmentation could be reduced by tuning GC parameters as described in [this post by Nate Berkopec](https://www.speedshop.co/2017/12/04/malloc-doubles-ruby-memory.html), which should be considered as a tradeoff, as it may affect overall performance of memory allocation and GC cycles.
+
 ## Importance of Changes
 
 When working on performance improvements, it's important to always ask yourself

doc/development/rake_tasks.md

@@ -143,12 +143,6 @@ This will compile and minify all JavaScript and CSS assets and copy them along
 with all other frontend assets (images, fonts, etc) into `/public/assets` where
 they can be easily inspected.
 
-## Generate API documentation for project services (e.g. Slack)
-
-```
-bundle exec rake services:doc
-```
-
 ## Updating Emoji Aliases
 
 To update the Emoji aliases file (used for Emoji autocomplete) you must run the

doc/user/application_security/sast/index.md

@@ -45,12 +45,15 @@ The results are sorted by the priority of the vulnerability:
 
 ## Requirements
 
-To run a SAST job, you need GitLab Runner with the
+To run a SAST job, by default, you need GitLab Runner with the
 [`docker`](https://docs.gitlab.com/runner/executors/docker.html#use-docker-in-docker-with-privileged-mode) or
 [`kubernetes`](https://docs.gitlab.com/runner/install/kubernetes.html#running-privileged-containers-for-the-runners)
 executor running in privileged mode. If you're using the shared Runners on GitLab.com,
 this is enabled by default.
 
+Privileged mode is not necessary if you've [disabled Docker in Docker
+for SAST](#disabling-docker-in-docker-for-sast)
+
 CAUTION: **Caution:**
 If you use your own Runners, make sure that the Docker version you have installed
 is **not** `19.03.00`. See [troubleshooting information](#error-response-from-daemon-error-processing-tar-file-docker-tar-relocation-error) for details.
@@ -144,6 +147,21 @@ under your project's settings:
 | ---- | --- | ----- |
 | Variable | `MAVEN_CLI_OPTS` | `-Drepository.password=verysecret -Drepository.user=myuser` |
 
+### Disabling Docker in Docker for SAST
+
+You can avoid the need for Docker in Docker by running the individual analyzers.
+This does not require running the executor in privileged mode. For example:
+
+```yaml
+include:
+  template: SAST.gitlab-ci.yml
+
+variables:
+  SAST_DISABLE_DIND: "true"
+```
+
+This will create individual `<analyzer-name>-sast` jobs for each analyzer that runs in your CI/CD pipeline.
+
 ### Overriding the SAST template
 
 If you want to override the job definition (for example, change properties like
@@ -173,6 +191,7 @@ The following are Docker image-related variables.
 | `SAST_ANALYZER_IMAGE_PREFIX`  | Override the name of the Docker registry providing the default images (proxy). Read more about [customizing analyzers](analyzers.md). |
 | `SAST_ANALYZER_IMAGE_TAG`     | Override the Docker tag of the default images. Read more about [customizing analyzers](analyzers.md).                                 |
 | `SAST_DEFAULT_ANALYZERS`      | Override the names of default images. Read more about [customizing analyzers](analyzers.md).                                          |
+| `SAST_DISABLE_DIND`           | Disable Docker in Docker and run analyzers [individually](#disabling-docker-in-docker-for-sast).                                      |
 | `SAST_PULL_ANALYZER_IMAGES`   | Pull the images from the Docker registry (set to 0 to disable). Read more about [customizing analyzers](analyzers.md).                 |
 
 ### Vulnerability filters

doc/user/project/integrations/generic_alerts.md

@@ -13,11 +13,11 @@ authored by the GitLab Alert Bot.
 
 ## Setting up generic alerts
 
-To set up the generic alerts integration:  
+To set up the generic alerts integration:
 
-1. Navigate to **Settings > Integrations** in a project.  
+1. Navigate to **Settings > Integrations** in a project.
 1. Click on **Alert endpoint**.
-1. Toggle the **Active**  alert setting. The `URL` and `Authorization Key` for the webhook configuration can be found there.  
+1. Toggle the **Active**  alert setting. The `URL` and `Authorization Key` for the webhook configuration can be found there.
 
 ## Customizing the payload
 
@@ -35,7 +35,11 @@ You can customize the payload by sending the following parameters. All fields ar
 Example request:
 
 ```sh
-curl --request POST --data '{"title": "Incident title"}' --header "Authorization: Bearer <autorization_key>" <url>
+curl --request POST \
+  --data '{"title": "Incident title"}' \
+  --header "Authorization: Bearer <autorization_key>" \
+  --header "Content-Type: application/json" \
+  <url>
 ```
 
 The `<autorization_key>` and `<url>` values can be found when [setting up generic alerts](#setting-up-generic-alerts).

lib/gitlab/ci/build/rules/rule/clause/exists.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Build
+      class Rules::Rule::Clause::Exists < Rules::Rule::Clause
+        # The maximum number of patterned glob comparisons that will be
+        # performed before the rule assumes that it has a match
+        MAX_PATTERN_COMPARISONS = 10_000
+
+        def initialize(globs)
+          globs = Array(globs)
+
+          @top_level_only = globs.all?(&method(:top_level_glob?))
+          @exact_globs, @pattern_globs = globs.partition(&method(:exact_glob?))
+        end
+
+        def satisfied_by?(pipeline, seed)
+          paths = worktree_paths(pipeline)
+
+          exact_matches?(paths) || pattern_matches?(paths)
+        end
+
+        private
+
+        def worktree_paths(pipeline)
+          if @top_level_only
+            pipeline.top_level_worktree_paths
+          else
+            pipeline.all_worktree_paths
+          end
+        end
+
+        def exact_matches?(paths)
+          @exact_globs.any? { |glob| paths.bsearch { |path| glob <=> path } }
+        end
+
+        def pattern_matches?(paths)
+          comparisons = 0
+          @pattern_globs.any? do |glob|
+            paths.any? do |path|
+              comparisons += 1
+              comparisons > MAX_PATTERN_COMPARISONS || pattern_match?(glob, path)
+            end
+          end
+        end
+
+        def pattern_match?(glob, path)
+          File.fnmatch?(glob, path, File::FNM_PATHNAME | File::FNM_DOTMATCH | File::FNM_EXTGLOB)
+        end
+
+        # matches glob patterns that only match files in the top level directory
+        def top_level_glob?(glob)
+          !glob.include?('/') && !glob.include?('**')
+        end
+
+        # matches glob patterns that have no metacharacters for File#fnmatch?
+        def exact_glob?(glob)
+          !glob.include?('*') && !glob.include?('?') && !glob.include?('[') && !glob.include?('{')
+        end
+      end
+    end
+  end
+end

lib/gitlab/ci/config/entry/rules/rule.rb

@@ -8,11 +8,11 @@ module Gitlab
           include ::Gitlab::Config::Entry::Validatable
           include ::Gitlab::Config::Entry::Attributable
 
-          CLAUSES      = %i[if changes].freeze
-          ALLOWED_KEYS = %i[if changes when start_in].freeze
+          CLAUSES      = %i[if changes exists].freeze
+          ALLOWED_KEYS = %i[if changes exists when start_in].freeze
           ALLOWED_WHEN = %w[on_success on_failure always never manual delayed].freeze
 
-          attributes :if, :changes, :when, :start_in
+          attributes :if, :changes, :exists, :when, :start_in
 
           validations do
             validates :config, presence: true
@@ -24,7 +24,7 @@ module Gitlab
 
             with_options allow_nil: true do
               validates :if, expression: true
-              validates :changes, array_of_strings: true
+              validates :changes, :exists, array_of_strings: true, length: { maximum: 50 }
               validates :when, allowed_values: { in: ALLOWED_WHEN }
             end
           end

lib/gitlab/danger/helper.rb

@@ -35,7 +35,8 @@ module Gitlab
       end
 
       def ee?
-        ENV['CI_PROJECT_NAME'] == 'gitlab' || File.exist?('../../CHANGELOG-EE.md')
+        # Support former project name for `dev` and support local Danger run
+        %w[gitlab gitlab-ee].include?(ENV['CI_PROJECT_NAME']) || Dir.exist?('../../ee')
       end
 
       def gitlab_helper

lib/tasks/services.rake

@@ -1,98 +0,0 @@
-services_template = <<-ERB
-# Services
-
-<% services.each  do |service| %>
-## <%= service[:title] %>
-
-
-<% unless  service[:description].blank? %>
-<%= service[:description] %>
-<% end  %>
-
-
-### Create/Edit <%= service[:title] %> service
-
-Set <%= service[:title] %> service for a project.
-<% unless  service[:help].blank? %>
-
-> <%= service[:help].gsub("\n", ' ') %>
-
-<% end  %>
-
-```
-PUT /projects/:id/services/<%= service[:dashed_name] %>
-
-```
-
-Parameters:
-
-<% service[:params].each  do |param| %>
-- `<%= param[:name] %>` <%= param[:required] ? "(**required**)" : "(optional)"  %><%= [" -",  param[:description]].join(" ").gsub("\n", '') unless param[:description].blank? %>
-
-<% end  %>
-
-### Delete <%= service[:title] %> service
-
-Delete <%= service[:title] %> service for a project.
-
-```
-DELETE /projects/:id/services/<%= service[:dashed_name] %>
-
-```
-
-### Get <%= service[:title] %> service settings
-
-Get <%= service[:title] %> service settings for a project.
-
-```
-GET /projects/:id/services/<%= service[:dashed_name] %>
-
-```
-
-<% end  %>
-ERB
-
-namespace :services do
-  task doc: :environment do
-    services = Service.available_services_names.map do |s|
-      service_start = Time.now
-      klass = "#{s}_service".classify.constantize
-
-      service = klass.new
-
-      service_hash = {}
-
-      service_hash[:title] = service.title
-      service_hash[:dashed_name] = s.dasherize
-      service_hash[:description] = service.description
-      service_hash[:help] = service.help
-      service_hash[:params] = service.fields.map do |p|
-        param_hash = {}
-
-        param_hash[:name] = p[:name]
-        param_hash[:description] = p[:placeholder] || p[:title]
-        param_hash[:required] = klass.validators_on(p[:name].to_sym).any? do |v|
-          v.class == ActiveRecord::Validations::PresenceValidator
-        end
-
-        param_hash
-      end
-      service_hash[:params].sort_by! { |p| p[:required] ? 0 : 1 }
-
-      puts "Collected data for: #{service.title}, #{Time.now - service_start}"
-      service_hash
-    end
-
-    doc_start = Time.now
-    doc_path = File.join(Rails.root, 'doc', 'api', 'services.md')
-
-    result = ERB.new(services_template, trim_mode: '>')
-      .result(OpenStruct.new(services: services).instance_eval { binding })
-
-    File.open(doc_path, 'w') do |f|
-      f.write result
-    end
-
-    puts "write a new service.md to: #{doc_path}, #{Time.now - doc_start}"
-  end
-end

locale/gitlab.pot

@@ -6550,6 +6550,9 @@ msgstr ""
 msgid "Faster as it re-uses the project workspace (falling back to clone if it doesn't exist)"
 msgstr ""
 
+msgid "Faster releases. Better code. Less pain."
+msgstr ""
+
 msgid "Favicon was successfully removed."
 msgstr ""
 
@@ -17682,6 +17685,9 @@ msgstr ""
 msgid "Weight %{weight}"
 msgstr ""
 
+msgid "Welcome to GitLab"
+msgstr ""
+
 msgid "Welcome to the Guided GitLab Tour"
 msgstr ""
 

scripts/review_apps/automated_cleanup.rb

@@ -18,7 +18,8 @@ class AutomatedCleanup
   ].freeze
 
   def self.ee?
-    ENV['CI_PROJECT_NAME'] == 'gitlab-ee' || File.exist?('CHANGELOG-EE.md')
+    # Support former project name for `dev`
+    %w[gitlab gitlab-ee].include?(ENV['CI_PROJECT_NAME'])
   end
 
   def initialize(project_path: ENV['CI_PROJECT_PATH'], gitlab_token: ENV['GITLAB_BOT_REVIEW_APPS_CLEANUP_TOKEN'])

scripts/trigger-build

@@ -12,7 +12,8 @@ end
 
 module Trigger
   def self.ee?
-    ENV['CI_PROJECT_NAME'] == 'gitlab-ee' || File.exist?('CHANGELOG-EE.md')
+    # Support former project name for `dev`
+    %w[gitlab gitlab-ee].include?(ENV['CI_PROJECT_NAME'])
   end
 
   class Base

spec/controllers/application_controller_spec.rb

@@ -94,14 +94,24 @@ describe ApplicationController do
           request.path = '/-/peek'
         end
 
-        it_behaves_like 'not setting gon variables'
+        # TODO:
+        # remove line below once `privacy_policy_update_callout`
+        # feature flag is removed and `gon` reverts back to
+        # to not setting any variables.
+        it_behaves_like 'setting gon variables'
+        # it_behaves_like 'not setting gon variables'
       end
     end
 
     context 'with json format' do
       let(:format) { :json }
 
-      it_behaves_like 'not setting gon variables'
+      # TODO:
+      # remove line below once `privacy_policy_update_callout`
+      # feature flag is removed and `gon` reverts back to
+      # to not setting any variables.
+      it_behaves_like 'setting gon variables'
+      # it_behaves_like 'not setting gon variables'
     end
   end
 

spec/lib/gitlab/ci/build/rules/rule/clause/changes_spec.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Ci::Build::Rules::Rule::Clause::Changes do
+  describe '#satisfied_by?' do
+    it_behaves_like 'a glob matching rule' do
+      let(:pipeline) { build(:ci_pipeline) }
+
+      before do
+        allow(pipeline).to receive(:modified_paths).and_return(files.keys)
+      end
+
+      subject { described_class.new(globs).satisfied_by?(pipeline, nil) }
+    end
+  end
+end

spec/lib/gitlab/ci/build/rules/rule/clause/exists_spec.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Ci::Build::Rules::Rule::Clause::Exists do
+  describe '#satisfied_by?' do
+    it_behaves_like 'a glob matching rule' do
+      let(:project) { create(:project, :custom_repo, files: files) }
+      let(:pipeline) { build(:ci_pipeline, project: project, sha: project.repository.head_commit.sha) }
+
+      subject { described_class.new(globs).satisfied_by?(pipeline, nil) }
+    end
+  end
+end

spec/lib/gitlab/ci/config/entry/rules/rule_spec.rb

@@ -103,6 +103,52 @@ describe Gitlab::Ci::Config::Entry::Rules::Rule do
       end
     end
 
+    context 'when using a long list as an invalid changes: clause' do
+      let(:config) { { changes: ['app/'] * 51 } }
+
+      it { is_expected.not_to be_valid }
+
+      it 'returns errors' do
+        expect(subject.errors).to include(/changes is too long \(maximum is 50 characters\)/)
+      end
+    end
+
+    context 'when using a exists: clause' do
+      let(:config) { { exists: %w[app/ lib/ spec/ other/* paths/**/*.rb] } }
+
+      it { is_expected.to be_valid }
+    end
+
+    context 'when using a string as an invalid exists: clause' do
+      let(:config) { { exists: 'a regular string' } }
+
+      it { is_expected.not_to be_valid }
+
+      it 'reports an error about invalid policy' do
+        expect(subject.errors).to include(/should be an array of strings/)
+      end
+    end
+
+    context 'when using a list as an invalid exists: clause' do
+      let(:config) { { exists: [1, 2] } }
+
+      it { is_expected.not_to be_valid }
+
+      it 'returns errors' do
+        expect(subject.errors).to include(/exists should be an array of strings/)
+      end
+    end
+
+    context 'when using a long list as an invalid exists: clause' do
+      let(:config) { { exists: ['app/'] * 51 } }
+
+      it { is_expected.not_to be_valid }
+
+      it 'returns errors' do
+        expect(subject.errors).to include(/exists is too long \(maximum is 50 characters\)/)
+      end
+    end
+
     context 'specifying a delayed job' do
       let(:config) { { if: '$THIS || $THAT', when: 'delayed', start_in: '15 minutes' } }
 
@@ -198,6 +244,12 @@ describe Gitlab::Ci::Config::Entry::Rules::Rule do
         expect(entry.value).to eq(config)
       end
     end
+
+    context 'when using a exists: clause' do
+      let(:config) { { exists: %w[app/ lib/ spec/ other/* paths/**/*.rb] } }
+
+      it { is_expected.to eq(config) }
+    end
   end
 
   describe '.default' do

spec/lib/gitlab/danger/helper_spec.rb

@@ -88,28 +88,28 @@ describe Gitlab::Danger::Helper do
 
     it 'returns true if CI_PROJECT_NAME if set to gitlab' do
       stub_env('CI_PROJECT_NAME', 'gitlab')
-      expect(File).not_to receive(:exist?)
+      expect(Dir).not_to receive(:exist?)
 
       is_expected.to be_truthy
     end
 
     it 'delegates to CHANGELOG-EE.md existence if CI_PROJECT_NAME is set to something else' do
       stub_env('CI_PROJECT_NAME', 'something else')
-      expect(File).to receive(:exist?).with('../../CHANGELOG-EE.md') { true }
+      expect(Dir).to receive(:exist?).with('../../ee') { true }
 
       is_expected.to be_truthy
     end
 
-    it 'returns true if CHANGELOG-EE.md exists' do
+    it 'returns true if ee exists' do
       stub_env('CI_PROJECT_NAME', nil)
-      expect(File).to receive(:exist?).with('../../CHANGELOG-EE.md') { true }
+      expect(Dir).to receive(:exist?).with('../../ee') { true }
 
       is_expected.to be_truthy
     end
 
-    it "returns false if CHANGELOG-EE.md doesn't exist" do
+    it "returns false if ee doesn't exist" do
       stub_env('CI_PROJECT_NAME', nil)
-      expect(File).to receive(:exist?).with('../../CHANGELOG-EE.md') { false }
+      expect(Dir).to receive(:exist?).with('../../ee') { false }
 
       is_expected.to be_falsy
     end

spec/models/ci/pipeline_spec.rb

@@ -1755,6 +1755,30 @@ describe Ci::Pipeline, :mailer do
     end
   end
 
+  describe '#all_worktree_paths' do
+    let(:files) { { 'main.go' => '', 'mocks/mocks.go' => '' } }
+    let(:project) { create(:project, :custom_repo, files: files) }
+    let(:pipeline) { build(:ci_pipeline, project: project, sha: project.repository.head_commit.sha) }
+
+    it 'returns all file paths cached' do
+      expect(project.repository).to receive(:ls_files).with(pipeline.sha).once.and_call_original
+      expect(pipeline.all_worktree_paths).to eq(files.keys)
+      expect(pipeline.all_worktree_paths).to eq(files.keys)
+    end
+  end
+
+  describe '#top_level_worktree_paths' do
+    let(:files) { { 'main.go' => '', 'mocks/mocks.go' => '' } }
+    let(:project) { create(:project, :custom_repo, files: files) }
+    let(:pipeline) { build(:ci_pipeline, project: project, sha: project.repository.head_commit.sha) }
+
+    it 'returns top-level file paths cached' do
+      expect(project.repository).to receive(:tree).with(pipeline.sha).once.and_call_original
+      expect(pipeline.top_level_worktree_paths).to eq(['main.go'])
+      expect(pipeline.top_level_worktree_paths).to eq(['main.go'])
+    end
+  end
+
   describe '#has_kubernetes_active?' do
     context 'when kubernetes is active' do
       context 'when user configured kubernetes from CI/CD > Clusters' do

spec/services/ci/create_pipeline_service/rules_spec.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Ci::CreatePipelineService do
+  context 'rules' do
+    let(:user)        { create(:admin) }
+    let(:ref)         { 'refs/heads/master' }
+    let(:source)      { :push }
+    let(:service)     { described_class.new(project, user, { ref: ref }) }
+    let(:pipeline)    { service.execute(source) }
+    let(:build_names) { pipeline.builds.pluck(:name) }
+
+    before do
+      stub_ci_pipeline_yaml_file(config)
+      allow_any_instance_of(Ci::BuildScheduleWorker).to receive(:perform).and_return(true)
+    end
+
+    context 'exists:' do
+      let(:config) do
+        <<-EOY
+        regular-job:
+          script: 'echo Hello, World!'
+
+        rules-job:
+          script: "echo hello world, $CI_COMMIT_REF_NAME"
+          rules:
+            - exists:
+              - README.md
+              when: manual
+            - exists:
+              - app.rb
+              when: on_success
+
+        delayed-job:
+          script: "echo See you later, World!"
+          rules:
+            - exists:
+              - README.md
+              when: delayed
+              start_in: 4 hours
+        EOY
+      end
+      let(:regular_job) { pipeline.builds.find_by(name: 'regular-job') }
+      let(:rules_job)   { pipeline.builds.find_by(name: 'rules-job') }
+      let(:delayed_job) { pipeline.builds.find_by(name: 'delayed-job') }
+
+      context 'with matches' do
+        let(:project) { create(:project, :custom_repo, files: { 'README.md' => '' }) }
+
+        it 'creates two jobs' do
+          expect(pipeline).to be_persisted
+          expect(build_names).to contain_exactly('regular-job', 'rules-job', 'delayed-job')
+        end
+
+        it 'sets when: for all jobs' do
+          expect(regular_job.when).to eq('on_success')
+          expect(rules_job.when).to eq('manual')
+          expect(delayed_job.when).to eq('delayed')
+          expect(delayed_job.options[:start_in]).to eq('4 hours')
+        end
+      end
+
+      context 'with matches on the second rule' do
+        let(:project) { create(:project, :custom_repo, files: { 'app.rb' => '' }) }
+
+        it 'includes both jobs' do
+          expect(pipeline).to be_persisted
+          expect(build_names).to contain_exactly('regular-job', 'rules-job')
+        end
+
+        it 'sets when: for the created rules job based on the second clause' do
+          expect(regular_job.when).to eq('on_success')
+          expect(rules_job.when).to eq('on_success')
+        end
+      end
+
+      context 'without matches' do
+        let(:project) { create(:project, :custom_repo, files: { 'useless_script.rb' => '' }) }
+
+        it 'only persists the job without rules' do
+          expect(pipeline).to be_persisted
+          expect(regular_job).to be_persisted
+          expect(rules_job).to be_nil
+          expect(delayed_job).to be_nil
+        end
+
+        it 'sets when: for the created job' do
+          expect(regular_job.when).to eq('on_success')
+        end
+      end
+    end
+  end
+end

spec/support/shared_examples/lib/gitlab/ci/build/rules/rule/clause/clause_shared_examples.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+RSpec.shared_examples 'a glob matching rule' do
+  using RSpec::Parameterized::TableSyntax
+
+  where(:case_name, :globs, :files, :satisfied) do
+    'exact top-level match'      | ['Dockerfile']               | { 'Dockerfile' => '', 'Gemfile' => '' }            | true
+    'exact top-level no match'   | ['Dockerfile']               | { 'Gemfile' => '' }                                | false
+    'pattern top-level match'    | ['Docker*']                  | { 'Dockerfile' => '', 'Gemfile' => '' }            | true
+    'pattern top-level no match' | ['Docker*']                  | { 'Gemfile' => '' }                                | false
+    'exact nested match'         | ['project/build.properties'] | { 'project/build.properties' => '' }               | true
+    'exact nested no match'      | ['project/build.properties'] | { 'project/README.md' => '' }                      | false
+    'pattern nested match'       | ['src/**/*.go']              | { 'src/gitlab.com/goproject/goproject.go' => '' }  | true
+    'pattern nested no match'    | ['src/**/*.go']              | { 'src/gitlab.com/goproject/README.md' => '' }     | false
+    'ext top-level match'        | ['*.go']                     | { 'main.go' => '', 'cmd/goproject/main.go' => '' } | true
+    'ext nested no match'        | ['*.go']                     | { 'cmd/goproject/main.go' => '' }                  | false
+    'ext slash no match'         | ['/*.go']                    | { 'main.go' => '', 'cmd/goproject/main.go' => '' } | false
+  end
+
+  with_them do
+    it { is_expected.to eq(satisfied) }
+  end
+end