From e67cd0407febc97c0676bea1f1e6f7739912d94f Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Mon, 14 Mar 2022 03:07:26 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- app/finders/group_members_finder.rb | 7 ++ app/models/application_record.rb | 4 + doc/administration/clusters/kas.md | 113 +++++++++--------- doc/user/clusters/agent/install/index.md | 3 +- lib/atlassian/jira_connect/client.rb | 25 +--- lib/atlassian/jira_connect/dev_info.rb | 48 ++++++++ .../api/schemas/jira_connect/dev_info.json | 8 ++ .../lib/atlassian/jira_connect/client_spec.rb | 20 ++-- .../atlassian/jira_connect/dev_info_spec.rb | 54 +++++++++ spec/models/application_record_spec.rb | 12 ++ 10 files changed, 202 insertions(+), 92 deletions(-) create mode 100644 lib/atlassian/jira_connect/dev_info.rb create mode 100644 spec/fixtures/api/schemas/jira_connect/dev_info.json create mode 100644 spec/lib/atlassian/jira_connect/dev_info_spec.rb diff --git a/app/finders/group_members_finder.rb b/app/finders/group_members_finder.rb index fff17098c7b..4213a3f1965 100644 --- a/app/finders/group_members_finder.rb +++ b/app/finders/group_members_finder.rb @@ -60,6 +60,8 @@ class GroupMembersFinder < UnionFinder members = members.filter_by_2fa(params[:two_factor]) end + members = apply_additional_filters(members) + by_created_at(members) end @@ -84,6 +86,11 @@ class GroupMembersFinder < UnionFinder raise ArgumentError, "#{(include_relations - RELATIONS).first} #{INVALID_RELATION_TYPE_ERROR_MSG}" end end + + def apply_additional_filters(members) + # overridden in EE to include additional filtering conditions. + members + end end GroupMembersFinder.prepend_mod_with('GroupMembersFinder') diff --git a/app/models/application_record.rb b/app/models/application_record.rb index 75e1bab294f..198a3653cd3 100644 --- a/app/models/application_record.rb +++ b/app/models/application_record.rb @@ -102,6 +102,10 @@ class ApplicationRecord < ActiveRecord::Base where('EXISTS (?)', query.select(1)) end + def self.where_not_exists(query) + where('NOT EXISTS (?)', query.select(1)) + end + def self.declarative_enum(enum_mod) enum(enum_mod.key => enum_mod.values) end diff --git a/doc/administration/clusters/kas.md b/doc/administration/clusters/kas.md index af31523e875..abc3ffa539e 100644 --- a/doc/administration/clusters/kas.md +++ b/doc/administration/clusters/kas.md @@ -6,32 +6,31 @@ info: To determine the technical writer assigned to the Stage/Group associated w # Install the GitLab agent server for Kubernetes (KAS) **(FREE SELF)** -> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3834) in GitLab 13.10, the GitLab agent server (KAS) became available on GitLab.com under `wss://kas.gitlab.com`. +> - [Introduced](https://gitlab.com/groups/gitlab-org/-/epics/3834) in GitLab 13.10, the GitLab agent server (KAS) became available on GitLab.com at `wss://kas.gitlab.com`. > - [Moved](https://gitlab.com/groups/gitlab-org/-/epics/6290) from GitLab Premium to GitLab Free in 14.5. -The GitLab agent server for Kubernetes is a GitLab backend service dedicated to -managing the [GitLab agent for Kubernetes](../../user/clusters/agent/index.md). +The GitLab agent server for Kubernetes (KAS) is a service that +manages the [GitLab agent for Kubernetes](../../user/clusters/agent/index.md). -The KAS acronym refers to the former name, Kubernetes agent server. +The KAS acronym refers to the former name, `Kubernetes agent server`. -The KAS is already installed and available in GitLab.com under `wss://kas.gitlab.com`. -This document describes how to install a KAS for GitLab self-managed instances. +The agent server for Kubernetes is installed and available on GitLab.com at `wss://kas.gitlab.com`. +If you use self-managed GitLab, you must install an agent server or specify an external installation. ## Installation options -As a GitLab administrator of self-managed instances, you can install KAS according to your GitLab -installation method: +As a GitLab administrator, you can install the agent server: -- For [Omnibus installations](#install-kas-with-omnibus). -- For [GitLab Helm Chart installations](#install-kas-with-the-gitlab-helm-chart). +- For [Omnibus installations](#for-omnibus). +- For [GitLab Helm Chart installations](#for-gitlab-helm-chart). -You can also opt to use an [external KAS](#use-an-external-kas-installation). +Or, you can [use an external agent server](#use-an-external-installation). -### Install KAS with Omnibus +### For Omnibus For [Omnibus](https://docs.gitlab.com/omnibus/) package installations: -1. Edit `/etc/gitlab/gitlab.rb` to enable the agent server: +1. To enable the agent server, edit `/etc/gitlab/gitlab.rb`: ```ruby gitlab_kas['enable'] = true @@ -39,49 +38,49 @@ For [Omnibus](https://docs.gitlab.com/omnibus/) package installations: 1. [Reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). -To configure any additional options related to your KAS, -refer to the **Enable GitLab KAS** section of the +For additional configuration options, see the **Enable GitLab KAS** section of the [`gitlab.rb.template`](https://gitlab.com/gitlab-org/omnibus-gitlab/-/blob/master/files/gitlab-config-template/gitlab.rb.template). -### Install KAS with the GitLab Helm Chart +### For GitLab Helm Chart -For GitLab [Helm Chart](https://docs.gitlab.com/charts/) -installations, you must set `global.kas.enabled` to `true`. -For example, in a shell with `helm` and `kubectl` -installed, run: +For GitLab [Helm Chart](https://docs.gitlab.com/charts/) installations: -```shell -helm repo add gitlab https://charts.gitlab.io/ -helm repo update -helm upgrade --install gitlab gitlab/gitlab \ - --timeout 600s \ - --set global.hosts.domain= \ - --set global.hosts.externalIP= \ - --set certmanager-issuer.email= \ - --set global.kas.enabled=true # <-- without this, KAS will not be installed -``` +1. Set `global.kas.enabled` to `true`. For example, in a shell with `helm` and `kubectl` + installed, run: -To configure KAS, use a `gitlab.kas` sub-section in your `values.yaml` file: + ```shell + helm repo add gitlab https://charts.gitlab.io/ + helm repo update + helm upgrade --install gitlab gitlab/gitlab \ + --timeout 600s \ + --set global.hosts.domain= \ + --set global.hosts.externalIP= \ + --set certmanager-issuer.email= \ + --set global.kas.enabled=true # <-- without this setting, the agent server will not be installed + ``` -```yaml -gitlab: - kas: - # put your KAS custom options here -``` +1. To configure the agent server, use a `gitlab.kas` sub-section in your `values.yaml` file: + + ```yaml + gitlab: + kas: + # put your custom options here + ``` For details, see [how to use the GitLab-KAS chart](https://docs.gitlab.com/charts/charts/gitlab/kas/). -### Use an external KAS installation +### Use an external installation > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/299850) in GitLab 13.10. -Besides installing KAS with GitLab, you can opt to configure GitLab to use an external KAS. +Instead of installing the agent server, you can configure GitLab to use an external agent server. -For GitLab instances installed through the GitLab Helm Chart, see [how to configure your external KAS](https://docs.gitlab.com/charts/charts/globals.html#external-kas). +If you used the GitLab Helm Chart to install GitLab, see +[how to configure your external agent server](https://docs.gitlab.com/charts/charts/globals.html#external-kas). -For GitLab instances installed through Omnibus packages: +If you used the Omnibus packages: -1. Edit `/etc/gitlab/gitlab.rb` adding the paths to your external KAS: +1. Edit `/etc/gitlab/gitlab.rb` and add the paths to your external agent server: ```ruby gitlab_kas['enable'] = false @@ -96,7 +95,7 @@ For GitLab instances installed through Omnibus packages: ## Troubleshooting -If you have issues while using the GitLab agent server for Kubernetes, view the +If you have issues while using the agent server for Kubernetes, view the service logs by running the following command: ```shell @@ -107,7 +106,7 @@ In Omnibus GitLab, find the logs in `/var/log/gitlab/gitlab-kas/`. You can also [troubleshoot issues with individual agents](../../user/clusters/agent/troubleshooting.md). -### KAS logs - GitOps: failed to get project information +### GitOps: failed to get project information If you get the following error message: @@ -115,11 +114,11 @@ If you get the following error message: {"level":"warn","time":"2020-10-30T08:37:26.123Z","msg":"GitOps: failed to get project info","agent_id":4,"project_id":"root/kas-manifest001","error":"error kind: 0; status: 404"} ``` -It means that the specified manifest project `root/kas-manifest001` -doesn't exist or the manifest project is private. To fix it, make sure the project path is correct -and its visibility is [set to public](../../public_access/public_access.md). +The project specified by the manifest (`root/kas-manifest001`) +doesn't exist or the project where the manifest is kept is private. To fix this issue, +ensure the project path is correct and that the project's visibility is [set to public](../../public_access/public_access.md). -### KAS logs - Configuration file not found +### Configuration file not found If you get the following error message: @@ -127,29 +126,29 @@ If you get the following error message: time="2020-10-29T04:44:14Z" level=warning msg="Config: failed to fetch" agent_id=2 error="configuration file not found: \".gitlab/agents/test-agent/config.yaml\ ``` -It means that the path to the configuration project is incorrect, -or the path to `config.yaml` inside the project is not valid. +The path is incorrect for either: -To fix this, ensure that the paths to the configuration repository and to the `config.yaml` file -are correct. +- The repository where the agent was registered. +- The agent configuration file. -### KAS logs - `dial tcp :443: connect: connection refused` +To fix this issue, ensure that the paths are correct. -If you are running a self-managed GitLab instance and: +### `dial tcp :443: connect: connection refused` + +If you are running self-managed GitLab and: - The instance isn't running behind an SSL-terminating proxy. - The instance doesn't have HTTPS configured on the GitLab instance itself. - The instance's hostname resolves locally to its internal IP address. -You may see the following error when the KAS tries to connect to the GitLab API: +When the agent server tries to connect to the GitLab API, the following error might occur: ```json {"level":"error","time":"2021-08-16T14:56:47.289Z","msg":"GetAgentInfo()","correlation_id":"01FD7QE35RXXXX8R47WZFBAXTN","grpc_service":"gitlab.agent.reverse_tunnel.rpc.ReverseTunnel","grpc_method":"Connect","error":"Get \"https://gitlab.example.com/api/v4/internal/kubernetes/agent_info\": dial tcp 172.17.0.4:443: connect: connection refused"} ``` -To fix this for [Omnibus](https://docs.gitlab.com/omnibus/) package installations, -set the following parameter in `/etc/gitlab/gitlab.rb` -(replacing `gitlab.example.com` with your GitLab instance's hostname): +To fix this issue for [Omnibus](https://docs.gitlab.com/omnibus/) package installations, +set the following parameter in `/etc/gitlab/gitlab.rb`. Replace `gitlab.example.com` with your GitLab instance's hostname: ```ruby gitlab_kas['gitlab_address'] = 'http://gitlab.example.com' diff --git a/doc/user/clusters/agent/install/index.md b/doc/user/clusters/agent/install/index.md index 869c9e33d13..81171431bc5 100644 --- a/doc/user/clusters/agent/install/index.md +++ b/doc/user/clusters/agent/install/index.md @@ -21,6 +21,7 @@ Before you can install the agent in your cluster, you need: - [Amazon Elastic Kubernetes Service (EKS)](https://docs.aws.amazon.com/eks/latest/userguide/getting-started.html) - [Digital Ocean](https://docs.digitalocean.com/products/kubernetes/quickstart/) - On self-managed GitLab instances, a GitLab administrator must set up the [agent server](../../../../administration/clusters/kas.md). + On GitLab.com, the agent server is available at `wss://kas.gitlab.com`. ## Installation steps @@ -122,7 +123,7 @@ Flags: --agent-token string Access token registered for agent --agent-version string Version of the agentk image to use (default "v14.8.1") -h, --help help for generate - --kas-address string GitLab Kubernetes Agent Server address + --kas-address string GitLab agent server for Kubernetes address --name-prefix string The prefix to use for names of Kubernetes objects --namespace string Kubernetes namespace to create resources in (default "gitlab-agent") --no-rbac Do not include corresponding Roles and RoleBindings for the agent service account diff --git a/lib/atlassian/jira_connect/client.rb b/lib/atlassian/jira_connect/client.rb index b8aa2cc8ea0..d38516e5288 100644 --- a/lib/atlassian/jira_connect/client.rb +++ b/lib/atlassian/jira_connect/client.rb @@ -14,14 +14,14 @@ module Atlassian def send_info(project:, update_sequence_id: nil, **args) common = { project: project, update_sequence_id: update_sequence_id } - dev_info = args.slice(:commits, :branches, :merge_requests) + dev_info = DevInfo.new(**common.merge(args.slice(:commits, :branches, :merge_requests))) build_info = args.slice(:pipelines) deploy_info = args.slice(:deployments) ff_info = args.slice(:feature_flags) responses = [] - responses << store_dev_info(**common, **dev_info) if dev_info.present? + responses << store_dev_info(dev_info) if dev_info.present? responses << store_build_info(**common, **build_info) if build_info.present? responses << store_deploy_info(**common, **deploy_info) if deploy_info.present? responses << store_ff_info(**common, **ff_info) if ff_info.present? @@ -93,17 +93,8 @@ module Atlassian handle_response(r, 'builds') { |data| errors(data, 'rejectedBuilds') } end - def store_dev_info(project:, commits: nil, branches: nil, merge_requests: nil, update_sequence_id: nil) - repo = ::Atlassian::JiraConnect::Serializers::RepositoryEntity.represent( - project, - commits: commits, - branches: branches, - merge_requests: merge_requests, - user_notes_count: user_notes_count(merge_requests), - update_sequence_id: update_sequence_id - ) - - post('/rest/devinfo/0.10/bulk', { repositories: [repo] }) + def store_dev_info(dev_info) + post(dev_info.url, dev_info.body) end def post(path, payload) @@ -157,14 +148,6 @@ module Atlassian { 'errorMessages' => messages } end - def user_notes_count(merge_requests) - return unless merge_requests - - Note.count_for_collection(merge_requests.map(&:id), 'MergeRequest').to_h do |count_group| - [count_group.noteable_id, count_group.count] - end - end - def jwt_token(http_method, uri) claims = Atlassian::Jwt.build_claims( Atlassian::JiraConnect.app_key, diff --git a/lib/atlassian/jira_connect/dev_info.rb b/lib/atlassian/jira_connect/dev_info.rb new file mode 100644 index 00000000000..90ccc1939d2 --- /dev/null +++ b/lib/atlassian/jira_connect/dev_info.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Atlassian + module JiraConnect + class DevInfo + URL = '/rest/devinfo/0.10/bulk' + + def initialize(project:, commits: nil, branches: nil, merge_requests: nil, update_sequence_id: nil) + @project = project + @commits = commits + @branches = branches + @merge_requests = merge_requests + @update_sequence_id = update_sequence_id + end + + def url + URL + end + + def body + repo = ::Atlassian::JiraConnect::Serializers::RepositoryEntity.represent( + @project, + commits: @commits, + branches: @branches, + merge_requests: @merge_requests, + user_notes_count: user_notes_count, + update_sequence_id: @update_sequence_id + ) + + { repositories: [repo] } + end + + def present? + [@commits, @branches, @merge_requests].any?(&:present?) + end + + private + + def user_notes_count + return unless @merge_requests + + Note.count_for_collection(@merge_requests.map(&:id), 'MergeRequest').to_h do |count_group| + [count_group.noteable_id, count_group.count] + end + end + end + end +end diff --git a/spec/fixtures/api/schemas/jira_connect/dev_info.json b/spec/fixtures/api/schemas/jira_connect/dev_info.json new file mode 100644 index 00000000000..98437353fde --- /dev/null +++ b/spec/fixtures/api/schemas/jira_connect/dev_info.json @@ -0,0 +1,8 @@ +{ + "repositories": { + "type": "array", + "items": { + "$ref": "./repository.json" + } + } +} diff --git a/spec/lib/atlassian/jira_connect/client_spec.rb b/spec/lib/atlassian/jira_connect/client_spec.rb index dd3130c78bf..1857a1431df 100644 --- a/spec/lib/atlassian/jira_connect/client_spec.rb +++ b/spec/lib/atlassian/jira_connect/client_spec.rb @@ -58,12 +58,16 @@ RSpec.describe Atlassian::JiraConnect::Client do deployments: :q ).and_return(:deploys_stored) - expect(subject).to receive(:store_dev_info).with( + expect(Atlassian::JiraConnect::DevInfo).to receive(:new).with( project: project, update_sequence_id: :x, commits: :a, branches: :b, merge_requests: :c + ).and_call_original + + expect(subject).to receive(:store_dev_info).with( + instance_of(Atlassian::JiraConnect::DevInfo) ).and_return(:dev_stored) args = { @@ -83,9 +87,7 @@ RSpec.describe Atlassian::JiraConnect::Client do it 'only calls methods that we need to call' do expect(subject).to receive(:store_dev_info).with( - project: project, - update_sequence_id: :x, - commits: :a + instance_of(Atlassian::JiraConnect::DevInfo) ).and_return(:dev_stored) args = { @@ -402,15 +404,7 @@ RSpec.describe Atlassian::JiraConnect::Client do end it "calls the API with auth headers" do - subject.send(:store_dev_info, project: project) - end - - it 'avoids N+1 database queries' do - control_count = ActiveRecord::QueryRecorder.new { subject.send(:store_dev_info, project: project, merge_requests: merge_requests) }.count - - merge_requests << create(:merge_request, :unique_branches) - - expect { subject.send(:store_dev_info, project: project, merge_requests: merge_requests) }.not_to exceed_query_limit(control_count) + subject.send(:store_dev_info, Atlassian::JiraConnect::DevInfo.new(project: project)) end end end diff --git a/spec/lib/atlassian/jira_connect/dev_info_spec.rb b/spec/lib/atlassian/jira_connect/dev_info_spec.rb new file mode 100644 index 00000000000..357168a94b9 --- /dev/null +++ b/spec/lib/atlassian/jira_connect/dev_info_spec.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Atlassian::JiraConnect::DevInfo do + let_it_be(:project) { create_default(:project, :repository).freeze } + + let(:update_sequence_id) { '123' } + + describe '#url' do + subject { described_class.new(project: project).url } + + it { is_expected.to eq('/rest/devinfo/0.10/bulk') } + end + + describe '#body' do + let_it_be(:merge_request) { create(:merge_request, :unique_branches, title: 'TEST-123') } + let_it_be(:note) { create(:note, noteable: merge_request, project: merge_request.project) } + let_it_be(:branches) do + project.repository.create_branch('TEST-123', project.default_branch_or_main) + [project.repository.find_branch('TEST-123')] + end + + let(:merge_requests) { [merge_request] } + + subject(:body) { described_class.new(project: project, branches: branches, merge_requests: merge_requests, update_sequence_id: update_sequence_id).body.to_json } + + it 'matches the schema' do + expect(body).to match_schema('jira_connect/dev_info') + end + + it 'avoids N+1 database queries' do + control_count = ActiveRecord::QueryRecorder.new { subject }.count + + merge_requests << create(:merge_request, :unique_branches) + + expect { subject }.not_to exceed_query_limit(control_count) + end + end + + describe '#present?' do + let(:arguments) { { commits: nil, branches: nil, merge_requests: nil } } + + subject { described_class.new(**{ project: project, update_sequence_id: update_sequence_id }.merge(arguments)).present? } + + it { is_expected.to eq(false) } + + context 'with commits, branches or merge requests' do + let(:arguments) { { commits: anything, branches: anything, merge_requests: anything } } + + it { is_expected.to eq(true) } + end + end +end diff --git a/spec/models/application_record_spec.rb b/spec/models/application_record_spec.rb index 9c9a048999c..c1cd44e9007 100644 --- a/spec/models/application_record_spec.rb +++ b/spec/models/application_record_spec.rb @@ -104,6 +104,18 @@ RSpec.describe ApplicationRecord do end end + describe '.where_not_exists' do + it 'produces a WHERE NOT EXISTS query' do + create(:user, :two_factor_via_u2f) + user_2 = create(:user) + + expect( + User.where_not_exists( + U2fRegistration.where(U2fRegistration.arel_table[:user_id].eq(User.arel_table[:id]))) + ).to match_array([user_2]) + end + end + describe '.transaction', :delete do it 'opens a new transaction' do expect(described_class.connection.transaction_open?).to be false