Add latest changes from gitlab-org/gitlab@master

This commit is contained in:
GitLab Bot 2022-01-18 18:11:20 +00:00
parent da975941d1
commit b53f926609
38 changed files with 524 additions and 307 deletions

View File

@ -45,7 +45,6 @@
# 10019, 10021 Missing security headers
# 10023, 10024, 10025, 10037 Information Disclosure
# 10040 Secure Pages Include Mixed Content
# 10055 CSP
# 10056 X-Debug-Token Information Leak
# Duration: 14 minutes 20 seconds
@ -54,7 +53,7 @@ dast:secureHeaders-csp-infoLeak:
- .dast_conf
variables:
DAST_USERNAME: "user1"
DAST_ONLY_INCLUDE_RULES: "10019,10021,10023,10024,10025,10037,10040,10055,10056"
DAST_ONLY_INCLUDE_RULES: "10019,10021,10023,10024,10025,10037,10040,10056"
script:
- /analyze

View File

@ -145,8 +145,7 @@ class Groups::DependencyProxyForContainersController < ::Groups::DependencyProxy
end
def dependency_proxy
@dependency_proxy ||=
group.dependency_proxy_setting || group.create_dependency_proxy_setting
@dependency_proxy ||= group.dependency_proxy_setting
end
def ensure_group

View File

@ -26,9 +26,13 @@ module Projects
).to_json
end
# @assignable_runners is using ci_owned_runners
::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336436') do
if current_user.ci_owned_runners_cross_joins_fix_enabled?
render
else
# @assignable_runners is using ci_owned_runners
::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336436') do
render
end
end
end

View File

@ -10,6 +10,10 @@ module Ci
where('traversal_ids @> ARRAY[?]::int[]', id)
end
scope :contains_any_of_namespaces, -> (ids) do
where('traversal_ids && ARRAY[?]::int[]', ids)
end
scope :by_namespace_id, -> (namespace_id) { where(namespace_id: namespace_id) }
class << self

View File

@ -776,6 +776,10 @@ class Group < Namespace
super || build_dependency_proxy_image_ttl_policy
end
def dependency_proxy_setting
super || build_dependency_proxy_setting
end
def crm_enabled?
crm_settings&.enabled?
end

View File

@ -1605,23 +1605,32 @@ class User < ApplicationRecord
def ci_owned_runners
@ci_owned_runners ||= begin
project_runners = Ci::RunnerProject
.where(project: authorized_projects(Gitlab::Access::MAINTAINER))
.joins(:runner)
.select('ci_runners.*')
group_runners = Ci::RunnerNamespace
.where(namespace_id: owned_groups.self_and_descendant_ids)
.joins(:runner)
.select('ci_runners.*')
Ci::Runner.from_union([project_runners, group_runners]).allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336436')
if ci_owned_runners_cross_joins_fix_enabled?
Ci::Runner
.from_union([ci_owned_project_runners_from_project_members,
ci_owned_project_runners_from_group_members,
ci_owned_group_runners])
else
Ci::Runner
.from_union([ci_legacy_owned_project_runners, ci_legacy_owned_group_runners])
.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336436')
end
end
end
def owns_runner?(runner)
::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336436') do
if ci_owned_runners_cross_joins_fix_enabled?
ci_owned_runners.exists?(runner.id)
else
::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/336436') do
ci_owned_runners.exists?(runner.id)
end
end
end
def ci_owned_runners_cross_joins_fix_enabled?
strong_memoize(:ci_owned_runners_cross_joins_fix_enabled) do
Feature.enabled?(:ci_owned_runners_cross_joins_fix, self, default_enabled: :yaml)
end
end
@ -2199,6 +2208,50 @@ class User < ApplicationRecord
::Gitlab::Auth::Ldap::Access.allowed?(self)
end
def ci_legacy_owned_project_runners
Ci::RunnerProject
.select('ci_runners.*')
.joins(:runner)
.where(project: authorized_projects(Gitlab::Access::MAINTAINER))
end
def ci_legacy_owned_group_runners
Ci::RunnerNamespace
.select('ci_runners.*')
.joins(:runner)
.where(namespace_id: owned_groups.self_and_descendant_ids)
end
def ci_owned_project_runners_from_project_members
Ci::RunnerProject
.select('ci_runners.*')
.joins(:runner)
.where(project: project_members.where('access_level >= ?', Gitlab::Access::MAINTAINER).pluck(:source_id))
end
def ci_owned_project_runners_from_group_members
Ci::RunnerProject
.select('ci_runners.*')
.joins(:runner)
.joins('JOIN ci_project_mirrors ON ci_project_mirrors.project_id = ci_runner_projects.project_id')
.joins('JOIN ci_namespace_mirrors ON ci_namespace_mirrors.namespace_id = ci_project_mirrors.namespace_id')
.merge(ci_namespace_mirrors_for_group_members(Gitlab::Access::MAINTAINER))
end
def ci_owned_group_runners
Ci::RunnerNamespace
.select('ci_runners.*')
.joins(:runner)
.joins('JOIN ci_namespace_mirrors ON ci_namespace_mirrors.namespace_id = ci_runner_namespaces.namespace_id')
.merge(ci_namespace_mirrors_for_group_members(Gitlab::Access::OWNER))
end
def ci_namespace_mirrors_for_group_members(level)
Ci::NamespaceMirror.contains_any_of_namespaces(
group_members.where('access_level >= ?', level).pluck(:source_id)
)
end
end
User.prepend_mod_with('User')

View File

@ -23,6 +23,8 @@ module Import
return ServiceResponse.error(message: "#{@params[:url]} is not a valid URL") unless uri
return ServiceResponse.success if uri.scheme == 'git'
uri.fragment = nil
url = Gitlab::Utils.append_path(uri.to_s, "/info/refs?service=#{GIT_SERVICE_NAME}")

View File

@ -0,0 +1,8 @@
---
name: ci_owned_runners_cross_joins_fix
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/78216
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/350322
milestone: '14.8'
type: development
group: group::pipeline execution
default_enabled: false

View File

@ -0,0 +1,8 @@
---
name: rate_limit_gitlab_shell
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/78373
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/350465
milestone: '14.7'
type: development
group: group::source code
default_enabled: false

View File

@ -5,4 +5,4 @@ rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/292199
milestone: '12.5'
type: development
group: group::editor
default_enabled: false
default_enabled: true

View File

@ -432,7 +432,16 @@ If large repositories are affected by this problem,
their resync may take a long time and cause significant load on your Geo nodes,
storage and network systems.
If you get the error message `Synchronization failed - Error syncing repository` along with the following log messages, this indicates that the expected `geo` remote is not present in the `.git/config` file
If you see the error message `Synchronization failed - Error syncing repository` along with `fatal: fsck error in packed object`, this indicates
a consistency check error when syncing the repository.
One example of a consistency error is: `error: object f4a87a3be694fbbd6e50a668a31a8513caeaafe3: hasDotgit: contains '.git`.
Removing the malformed objects causing consistency errors require rewriting the repository history, which is not always an option. However,
it's possible to override the consistency checks instead. To do that, follow
[the instructions in the Gitaly docs](../../gitaly/configure_gitaly.md#repository-consistency-checks).
You can also get the error message `Synchronization failed - Error syncing repository` along with the following log messages, this indicates that the expected `geo` remote is not present in the `.git/config` file
of a repository on the secondary Geo node's file system:
```json

View File

@ -32,10 +32,18 @@ Two things need to be configured for the interactive web terminal to work:
- If you are using a reverse proxy with your GitLab instance, web terminals need to be
[enabled](../../administration/integration/terminal.md#enabling-and-disabling-terminal-support)
NOTE:
Interactive web terminals are not yet supported by
[`gitlab-runner` Helm chart](https://docs.gitlab.com/charts/charts/gitlab/gitlab-runner/index.html).
Support is tracked [in this issue](https://gitlab.com/gitlab-org/charts/gitlab-runner/-/issues/79).
### Partial support for Helm chart
Interactive web terminals are partially supported in `gitlab-runner` Helm chart.
They are enabled when:
- The number of replica is one
- You use the `loadBalancer` service
Support for fixing these limitations is tracked in the following issues:
- [Support of more than one replica](https://gitlab.com/gitlab-org/charts/gitlab-runner/-/issues/323)
- [Support of more service types](https://gitlab.com/gitlab-org/charts/gitlab-runner/-/issues/324)
## Debugging a running job

View File

@ -7,8 +7,13 @@ type: reference, how-to
# Sourcegraph integration **(FREE)**
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/16556) in GitLab 12.5.
> - Note that this integration is in BETA and deployed [behind a feature flag](#enable-the-sourcegraph-feature-flag) disabled by default. Self-managed instances can opt to enable it.
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/16556) in GitLab 12.5 [with a flag](../administration/feature_flags.md) named `sourcegraph`. Disabled by default.
> - Enabled on GitLab.com in GitLab 12.5.
> - [Enabled on self-managed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/73116) in GitLab 14.8.
FLAG:
On self-managed GitLab, by default this feature is available. To hide the feature, ask an administrator to [disable the feature flag](../administration/feature_flags.md) named `sourcegraph`.
On GitLab.com, this feature is available for public projects only.
[Sourcegraph](https://sourcegraph.com) provides code intelligence features, natively integrated into the GitLab UI.
@ -26,41 +31,9 @@ you can choose to enable Sourcegraph [through your user preferences](#enable-sou
## Set up for self-managed GitLab instances **(FREE SELF)**
Before you can enable Sourcegraph code intelligence in GitLab you must:
configure a Sourcegraph instance with your GitLab instance as an external service.
- Enable the `sourcegraph` feature flag for your GitLab instance.
- Configure a Sourcegraph instance with your GitLab instance as an external service.
### Enable the Sourcegraph feature flag
NOTE:
If you are running a self-managed instance, the Sourcegraph integration is unavailable
unless the feature flag `sourcegraph` is enabled. This can be done from the Rails console
by instance administrators.
Use these commands to start the Rails console:
```shell
# Omnibus GitLab
gitlab-rails console
# Installation from source
cd /home/git/gitlab
sudo -u git -H bin/rails console -e production
```
Then run the following command to enable the feature flag:
```ruby
Feature.enable(:sourcegraph)
```
You can also enable the feature flag only for specific projects with:
```ruby
Feature.enable(:sourcegraph, Project.find_by_full_path('my_group/my_project'))
```
### Set up a self-managed Sourcegraph instance
### Set up a self-managed Sourcegraph instance **(FREE SELF)**
If you are new to Sourcegraph, head over to the [Sourcegraph installation documentation](https://docs.sourcegraph.com/admin) and get your instance up and running.

View File

@ -267,10 +267,17 @@ previous period), log in to the [Customers Portal](https://customers.gitlab.com/
If you have difficulty during the renewal process, contact the
[Support team](https://support.gitlab.com/hc/en-us/requests/new?ticket_form_id=360000071293) for assistance.
## Change the contact person for your subscription
## Add or change the contacts for your subscription
To change the contact person who manages your subscription,
contact the GitLab [Support team](https://support.gitlab.com/hc/en-us/requests/new?ticket_form_id=360000071293).
Contacts can renew a subscription, cancel a subscription, or transfer the subscription to a different namespace.
To change the contacts:
1. Ensure an account exists in the
[Customers Portal](https://customers.gitlab.com/customers/sign_in) for the user you want to add.
1. Verify you have access to at least one of
[these requirements](https://about.gitlab.com/handbook/support/license-and-renewals/workflows/customersdot/associating_purchases.html).
1. [Create a ticket with the Support team](https://support.gitlab.com/hc/en-us/requests/new?ticket_form_id=360000071293). Include any relevant material in your request.
## CI/CD minutes

View File

@ -419,6 +419,18 @@ The following is emailed to you:
[Upload the new license](../../user/admin_area/license.md#upload-your-license) to your instance.
The new tier takes effect when the new license is uploaded.
## Add or change the contacts for your subscription
Contacts can renew a subscription, cancel a subscription, or transfer the subscription to a different namespace.
To change the contacts:
1. Ensure an account exists in the
[Customers Portal](https://customers.gitlab.com/customers/sign_in) for the user you want to add.
1. Verify you have access to at least one of
[these requirements](https://about.gitlab.com/handbook/support/license-and-renewals/workflows/customersdot/associating_purchases.html).
1. [Create a ticket with the Support team](https://support.gitlab.com/hc/en-us/requests/new?ticket_form_id=360000071293). Include any relevant material in your request.
## Subscription expiry
When your license expires, GitLab locks down features, like Git pushes

View File

@ -43,6 +43,10 @@ module API
# This is a separate method so that EE can alter its behaviour more
# easily.
if Feature.enabled?(:rate_limit_gitlab_shell, default_enabled: :yaml)
check_rate_limit!(:gitlab_shell_operation, scope: [params[:action], params[:project], actor.key_or_user])
end
# Stores some Git-specific env thread-safely
env = parse_env
Gitlab::Git::HookEnv.set(gl_repository, env) if container

View File

@ -56,7 +56,8 @@ module Gitlab
profile_update_username: { threshold: 10, interval: 1.minute },
update_environment_canary_ingress: { threshold: 1, interval: 1.minute },
auto_rollback_deployment: { threshold: 1, interval: 3.minutes },
user_email_lookup: { threshold: -> { application_settings.user_email_lookup_limit }, interval: 1.minute }
user_email_lookup: { threshold: -> { application_settings.user_email_lookup_limit }, interval: 1.minute },
gitlab_shell_operation: { threshold: 600, interval: 1.minute }
}.freeze
end
@ -64,7 +65,7 @@ module Gitlab
# be throttled.
#
# @param key [Symbol] Key attribute registered in `.rate_limits`
# @param scope [Array<ActiveRecord>] Array of ActiveRecord models to scope throttling to a specific request (e.g. per user per project)
# @param scope [Array<ActiveRecord>] Array of ActiveRecord models, Strings or Symbols to scope throttling to a specific request (e.g. per user per project)
# @param threshold [Integer] Optional threshold value to override default one registered in `.rate_limits`
# @param users_allowlist [Array<String>] Optional list of usernames to exclude from the limit. This param will only be functional if Scope includes a current user.
# @param peek [Boolean] Optional. When true the key will not be incremented but the current throttled state will be returned.

View File

@ -8,13 +8,14 @@ module Gitlab
end
def feature_available?
# The sourcegraph_bundle feature could be conditionally applied, so check if `!off?`
!feature.off?
# The sourcegraph feature could be conditionally applied, so check if `!off?`
# We also can't just check !off? because the ActiveRecord might not exist yet
self.feature_enabled? || !feature.off?
end
def feature_enabled?(actor = nil)
# Some CI jobs grep for Feature.enabled? in our codebase, so it is important this reference stays around.
Feature.enabled?(:sourcegraph, actor)
Feature.enabled?(:sourcegraph, actor, default_enabled: :yaml)
end
private

View File

@ -2,41 +2,6 @@
namespace :gitlab do
namespace :gitaly do
desc 'Installs gitaly for running tests within gitlab-development-kit'
task :test_install, [:dir, :storage_path, :repo] => :gitlab_environment do |t, args|
inside_gdk = Rails.env.test? && File.exist?(Rails.root.join('../GDK_ROOT'))
if ENV['FORCE_GITALY_INSTALL'] || !inside_gdk
Rake::Task["gitlab:gitaly:install"].invoke(*args)
next
end
gdk_gitaly_dir = ENV.fetch('GDK_GITALY', Rails.root.join('../gitaly'))
# Our test setup expects a git repo, so clone rather than copy
clone_repo(gdk_gitaly_dir, args.dir, clone_opts: %w[--depth 1]) unless Dir.exist?(args.dir)
# We assume the GDK gitaly already compiled binaries
build_dir = File.join(gdk_gitaly_dir, '_build')
FileUtils.cp_r(build_dir, args.dir)
# We assume the GDK gitaly already ran bundle install
bundle_dir = File.join(gdk_gitaly_dir, 'ruby', '.bundle')
FileUtils.cp_r(bundle_dir, File.join(args.dir, 'ruby'))
# For completeness we copy this for gitaly's make target
ruby_bundle_file = File.join(gdk_gitaly_dir, '.ruby-bundle')
FileUtils.cp_r(ruby_bundle_file, args.dir)
gitaly_binary = File.join(build_dir, 'bin', 'gitaly')
warn_gitaly_out_of_date!(gitaly_binary, Gitlab::GitalyClient.expected_server_version)
rescue Errno::ENOENT => e
puts "Could not copy files, did you run `gdk update`? Error: #{e.message}"
raise
end
desc 'GitLab | Gitaly | Clone and checkout gitaly'
task :clone, [:dir, :storage_path, :repo] => :gitlab_environment do |t, args|
warn_user_is_not_gitlab
@ -60,9 +25,6 @@ Usage: rake "gitlab:gitaly:install[/installation/dir,/storage/path]")
storage_paths = { 'default' => args.storage_path }
Gitlab::SetupHelper::Gitaly.create_configuration(args.dir, storage_paths)
# In CI we run scripts/gitaly-test-build
next if ENV['CI'].present?
Dir.chdir(args.dir) do
Bundler.with_original_env do
env = { "RUBYOPT" => nil, "BUNDLE_GEMFILE" => nil }

View File

@ -1,7 +1,7 @@
# frozen_string_literal: true
module QA
RSpec.describe 'Package Registry', :orchestrated, :packages, :reliable, :object_storage do
RSpec.describe 'Package Registry', :orchestrated, :packages, :object_storage do
describe 'npm instance level endpoint' do
using RSpec::Parameterized::TableSyntax
include Runtime::Fixtures

View File

@ -1,7 +1,7 @@
# frozen_string_literal: true
module QA
RSpec.describe 'Package Registry', :orchestrated, :packages, :reliable, :object_storage do
RSpec.describe 'Package Registry', :orchestrated, :packages, :object_storage do
describe 'npm project level endpoint' do
using RSpec::Parameterized::TableSyntax
include Runtime::Fixtures

View File

@ -13,8 +13,6 @@ class GitalyTestBuild
include GitalySetup
def run
set_bundler_config
# If we have the binaries from the cache, we can skip building them again
if File.exist?(tmp_tests_gitaly_bin_dir)
GitalySetup::LOGGER.debug "Gitaly binary already built. Skip building...\n"

View File

@ -9,17 +9,11 @@ class GitalyTestSpawn
include GitalySetup
def run
set_bundler_config
install_gitaly_gems if ENV['CI']
check_gitaly_config!
install_gitaly_gems
# # Uncomment line below to see all gitaly logs merged into CI trace
# spawn('sleep 1; tail -f log/gitaly-test.log')
# In local development this pid file is used by rspec.
IO.write(File.expand_path('../tmp/tests/gitaly.pid', __dir__), start_gitaly)
IO.write(File.expand_path('../tmp/tests/gitaly2.pid', __dir__), start_gitaly2)
IO.write(File.expand_path('../tmp/tests/praefect.pid', __dir__), start_praefect)
# Optionally specify the path to the gitaly config toml as first argument.
# Used by workhorse in test.
spawn_gitaly(ARGV[0])
end
end

View File

@ -178,10 +178,6 @@ RSpec.describe Groups::DependencyProxyForContainersController do
subject { get_manifest(tag) }
context 'feature enabled' do
before do
enable_dependency_proxy
end
it_behaves_like 'without a token'
it_behaves_like 'without permission'
it_behaves_like 'feature flag disabled with private group'
@ -270,7 +266,6 @@ RSpec.describe Groups::DependencyProxyForContainersController do
let_it_be_with_reload(:group) { create(:group, parent: parent_group) }
before do
parent_group.create_dependency_proxy_setting!(enabled: true)
group_deploy_token.update_column(:group_id, parent_group.id)
end
@ -294,10 +289,6 @@ RSpec.describe Groups::DependencyProxyForContainersController do
subject { get_blob }
context 'feature enabled' do
before do
enable_dependency_proxy
end
it_behaves_like 'without a token'
it_behaves_like 'without permission'
it_behaves_like 'feature flag disabled with private group'
@ -341,7 +332,6 @@ RSpec.describe Groups::DependencyProxyForContainersController do
let_it_be_with_reload(:group) { create(:group, parent: parent_group) }
before do
parent_group.create_dependency_proxy_setting!(enabled: true)
group_deploy_token.update_column(:group_id, parent_group.id)
end
@ -474,10 +464,6 @@ RSpec.describe Groups::DependencyProxyForContainersController do
end
end
def enable_dependency_proxy
group.create_dependency_proxy_setting!(enabled: true)
end
def disable_dependency_proxy
group.create_dependency_proxy_setting!(enabled: false)
end

View File

@ -25,6 +25,19 @@ RSpec.describe Projects::Settings::CiCdController do
expect(response).to render_template(:show)
end
context 'when the FF ci_owned_runners_cross_joins_fix is disabled' do
before do
stub_feature_flags(ci_owned_runners_cross_joins_fix: false)
end
it 'renders show with 200 status code' do
get :show, params: { namespace_id: project.namespace, project_id: project }
expect(response).to have_gitlab_http_status(:ok)
expect(response).to render_template(:show)
end
end
context 'with CI/CD disabled' do
before do
project.project_feature.update_attribute(:builds_access_level, ProjectFeature::DISABLED)

View File

@ -5,9 +5,9 @@ require 'spec_helper'
RSpec.describe Mutations::Ci::Runner::Delete do
include GraphqlHelpers
let_it_be(:user) { create(:user) }
let_it_be(:runner) { create(:ci_runner) }
let(:user) { create(:user) }
let(:current_ctx) { { current_user: user } }
let(:mutation_params) do
@ -46,10 +46,10 @@ RSpec.describe Mutations::Ci::Runner::Delete do
end
context 'when user can delete owned runner' do
let_it_be(:project) { create(:project, creator_id: user.id) }
let_it_be(:project_runner, reload: true) { create(:ci_runner, :project, description: 'Project runner', projects: [project]) }
let!(:project) { create(:project, creator_id: user.id) }
let!(:project_runner) { create(:ci_runner, :project, description: 'Project runner', projects: [project]) }
before_all do
before do
project.add_maintainer(user)
end
@ -63,10 +63,10 @@ RSpec.describe Mutations::Ci::Runner::Delete do
end
context 'with more than one associated project' do
let_it_be(:project2) { create(:project, creator_id: user.id) }
let_it_be(:two_projects_runner) { create(:ci_runner, :project, description: 'Two projects runner', projects: [project, project2]) }
let!(:project2) { create(:project, creator_id: user.id) }
let!(:two_projects_runner) { create(:ci_runner, :project, description: 'Two projects runner', projects: [project, project2]) }
before_all do
before do
project2.add_maintainer(user)
end

View File

@ -37,6 +37,12 @@ RSpec.describe Gitlab::Sourcegraph do
it { is_expected.to be_truthy }
end
context 'when feature is disabled' do
let(:feature_scope) { false }
it { is_expected.to be_falsey }
end
end
describe '.feature_enabled?' do

View File

@ -30,6 +30,20 @@ RSpec.describe Ci::NamespaceMirror do
end
end
describe '.contains_any_of_namespaces' do
let!(:other_group1) { create(:group) }
let!(:other_group2) { create(:group, parent: other_group1) }
let!(:other_group3) { create(:group, parent: other_group2) }
subject(:result) { described_class.contains_any_of_namespaces([group2.id, other_group2.id]) }
it 'returns groups having group2.id in traversal_ids' do
expect(result.pluck(:namespace_id)).to contain_exactly(
group2.id, group3.id, group4.id, other_group2.id, other_group3.id
)
end
end
describe '.by_namespace_id' do
subject(:result) { described_class.by_namespace_id(group2.id) }

View File

@ -2823,6 +2823,26 @@ RSpec.describe Group do
end
end
describe '#dependency_proxy_setting' do
subject(:setting) { group.dependency_proxy_setting }
it 'builds a new policy if one does not exist', :aggregate_failures do
expect(setting.enabled).to eq(true)
expect(setting).not_to be_persisted
end
context 'with existing policy' do
before do
group.dependency_proxy_setting.update!(enabled: false)
end
it 'returns the policy if it already exists', :aggregate_failures do
expect(setting.enabled).to eq(false)
expect(setting).to be_persisted
end
end
end
describe '#crm_enabled?' do
it 'returns false where no crm_settings exist' do
expect(group.crm_enabled?).to be_falsey

View File

@ -3967,7 +3967,7 @@ RSpec.describe User do
end
end
describe '#ci_owned_runners' do
shared_context '#ci_owned_runners' do
let(:user) { create(:user) }
shared_examples :nested_groups_owner do
@ -4274,6 +4274,16 @@ RSpec.describe User do
end
end
it_behaves_like '#ci_owned_runners'
context 'when FF ci_owned_runners_cross_joins_fix is disabled' do
before do
stub_feature_flags(ci_owned_runners_cross_joins_fix: false)
end
it_behaves_like '#ci_owned_runners'
end
describe '#projects_with_reporter_access_limited_to' do
let(:project1) { create(:project) }
let(:project2) { create(:project) }

View File

@ -909,7 +909,6 @@ RSpec.describe GroupPolicy do
context 'feature enabled' do
before do
stub_config(dependency_proxy: { enabled: true })
group.create_dependency_proxy_setting!(enabled: true)
end
context 'reporter' do
@ -955,7 +954,6 @@ RSpec.describe GroupPolicy do
before do
stub_config(dependency_proxy: { enabled: true })
group.create_dependency_proxy_setting!(enabled: true)
end
it { is_expected.to be_allowed(:read_dependency_proxy) }

View File

@ -372,7 +372,38 @@ RSpec.describe API::Internal::Base do
end
end
describe "POST /internal/allowed", :clean_gitlab_redis_shared_state do
describe "POST /internal/allowed", :clean_gitlab_redis_shared_state, :clean_gitlab_redis_rate_limiting do
shared_examples 'rate limited request' do
let(:action) { 'git-upload-pack' }
let(:actor) { key }
it 'is throttled by rate limiter' do
allow(::Gitlab::ApplicationRateLimiter).to receive(:threshold).and_return(1)
expect(::Gitlab::ApplicationRateLimiter).to receive(:throttled?).with(:gitlab_shell_operation, scope: [action, project.full_path, actor]).twice.and_call_original
request
expect(response).to have_gitlab_http_status(:ok)
request
expect(response).to have_gitlab_http_status(:too_many_requests)
expect(json_response['message']['error']).to eq('This endpoint has been requested too many times. Try again later.')
end
context 'when rate_limit_gitlab_shell feature flag is disabled' do
before do
stub_feature_flags(rate_limit_gitlab_shell: false)
end
it 'is not throttled by rate limiter' do
expect(::Gitlab::ApplicationRateLimiter).not_to receive(:throttled?)
subject
end
end
end
context "access granted" do
let(:env) { {} }
@ -530,6 +561,32 @@ RSpec.describe API::Internal::Base do
expect(json_response["gitaly"]["features"]).to eq('gitaly-feature-mep-mep' => 'true')
expect(user.reload.last_activity_on).to eql(Date.today)
end
it_behaves_like 'rate limited request' do
def request
pull(key, project)
end
end
context 'when user_id is passed' do
it_behaves_like 'rate limited request' do
let(:actor) { user }
def request
post(
api("/internal/allowed"),
params: {
user_id: user.id,
project: full_path_for(project),
gl_repository: gl_repository_for(project),
action: 'git-upload-pack',
secret_token: secret_token,
protocol: 'ssh'
}
)
end
end
end
end
context "with a feature flag enabled for a project" do
@ -576,6 +633,14 @@ RSpec.describe API::Internal::Base do
expect(json_response["gitaly"]["token"]).to eq(Gitlab::GitalyClient.token(project.repository_storage))
expect(user.reload.last_activity_on).to be_nil
end
it_behaves_like 'rate limited request' do
let(:action) { 'git-receive-pack' }
def request
push(key, project)
end
end
end
context 'when receive_max_input_size has been updated' do
@ -838,6 +903,14 @@ RSpec.describe API::Internal::Base do
expect(json_response["gitaly"]["address"]).to eq(Gitlab::GitalyClient.address(project.repository_storage))
expect(json_response["gitaly"]["token"]).to eq(Gitlab::GitalyClient.token(project.repository_storage))
end
it_behaves_like 'rate limited request' do
let(:action) { 'git-upload-archive' }
def request
archive(key, project)
end
end
end
context "not added to project" do

View File

@ -499,9 +499,7 @@ RSpec.describe 'Rack Attack global throttles', :use_clean_rails_memory_store_cac
before do
group.add_owner(user)
group.create_dependency_proxy_setting!(enabled: true)
other_group.add_owner(other_user)
other_group.create_dependency_proxy_setting!(enabled: true)
allow(Gitlab.config.dependency_proxy)
.to receive(:enabled).and_return(true)

View File

@ -24,6 +24,17 @@ RSpec.describe Import::ValidateRemoteGitEndpointService do
expect(Gitlab::HTTP).to have_received(:get).with(endpoint_url, basic_auth: nil, stream_body: true, follow_redirects: false)
end
context 'when uri is using git:// protocol' do
subject { described_class.new(url: 'git://demo.host/repo')}
it 'returns success' do
result = subject.execute
expect(result).to be_a(ServiceResponse)
expect(result.success?).to be(true)
end
end
context 'when receiving HTTP response' do
subject { described_class.new(url: base_url) }

View File

@ -9,8 +9,13 @@
require 'securerandom'
require 'socket'
require 'logger'
require 'bundler'
module GitalySetup
extend self
REPOS_STORAGE = 'default'
LOGGER = begin
default_name = ENV['CI'] ? 'DEBUG' : 'WARN'
level_name = ENV['GITLAB_TESTING_LOG_LEVEL']&.upcase
@ -52,11 +57,13 @@ module GitalySetup
def env
{
'HOME' => expand_path('tmp/tests'),
'GEM_PATH' => Gem.path.join(':'),
'BUNDLE_APP_CONFIG' => File.join(gemfile_dir, '.bundle'),
'BUNDLE_INSTALL_FLAGS' => nil,
'BUNDLE_IGNORE_CONFIG' => '1',
'BUNDLE_PATH' => bundle_path,
'BUNDLE_GEMFILE' => gemfile,
'BUNDLE_JOBS' => '4',
'BUNDLE_RETRY' => '3',
'RUBYOPT' => nil,
# Git hooks can't run during tests as the internal API is not running.
@ -65,17 +72,20 @@ module GitalySetup
}
end
# rubocop:disable GitlabSecurity/SystemCommandInjection
def set_bundler_config
system('bundle config set --local jobs 4', chdir: gemfile_dir)
system('bundle config set --local retry 3', chdir: gemfile_dir)
def bundle_path
# Allow the user to override BUNDLE_PATH if they need to
return ENV['GITALY_TEST_BUNDLE_PATH'] if ENV['GITALY_TEST_BUNDLE_PATH']
if ENV['CI']
bundle_path = expand_path('vendor/gitaly-ruby')
system('bundle', 'config', 'set', '--local', 'path', bundle_path, chdir: gemfile_dir)
expand_path('vendor/gitaly-ruby')
else
explicit_path = Bundler.configured_bundle_path.explicit_path
return unless explicit_path
expand_path(explicit_path)
end
end
# rubocop:enable GitlabSecurity/SystemCommandInjection
def config_path(service)
case service
@ -88,6 +98,10 @@ module GitalySetup
end
end
def repos_path(storage = REPOS_STORAGE)
Gitlab.config.repositories.storages[REPOS_STORAGE].legacy_disk_path
end
def service_binary(service)
case service
when :gitaly, :gitaly2
@ -97,16 +111,20 @@ module GitalySetup
end
end
def run_command(cmd, env: {})
system(env, *cmd, exception: true, chdir: tmp_tests_gitaly_dir)
end
def install_gitaly_gems
system(env, "make #{tmp_tests_gitaly_dir}/.ruby-bundle", chdir: tmp_tests_gitaly_dir) # rubocop:disable GitlabSecurity/SystemCommandInjection
run_command(%W[make #{tmp_tests_gitaly_dir}/.ruby-bundle], env: env)
end
def build_gitaly
system(env.merge({ 'GIT_VERSION' => nil }), 'make all git', chdir: tmp_tests_gitaly_dir) # rubocop:disable GitlabSecurity/SystemCommandInjection
run_command(%w[make all git], env: env.merge('GIT_VERSION' => nil))
end
def start_gitaly
start(:gitaly)
def start_gitaly(toml = nil)
start(:gitaly, toml)
end
def start_gitaly2
@ -117,14 +135,20 @@ module GitalySetup
start(:praefect)
end
def start(service)
def start(service, toml = nil)
toml ||= config_path(service)
args = ["#{tmp_tests_gitaly_bin_dir}/#{service_binary(service)}"]
args.push("-config") if service == :praefect
args.push(config_path(service))
args.push(toml)
# Ensure user configuration does not affect Git
# Context: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/58776#note_547613780
env = self.env.merge('HOME' => nil, 'XDG_CONFIG_HOME' => nil)
pid = spawn(env, *args, [:out, :err] => "log/#{service}-test.log")
begin
try_connect!(service)
try_connect!(service, toml)
rescue StandardError
Process.kill('TERM', pid)
raise
@ -161,29 +185,37 @@ module GitalySetup
abort 'bundle check failed' unless system(env, 'bundle', 'check', out: out, chdir: gemfile_dir)
end
def read_socket_path(service)
def connect_proc(toml)
# This code needs to work in an environment where we cannot use bundler,
# so we cannot easily use the toml-rb gem. This ad-hoc parser should be
# good enough.
config_text = IO.read(config_path(service))
config_text = IO.read(toml)
config_text.lines.each do |line|
match_data = line.match(/^\s*socket_path\s*=\s*"([^"]*)"$/)
match_data = line.match(/^\s*(socket_path|listen_addr)\s*=\s*"([^"]*)"$/)
return match_data[1] if match_data
next unless match_data
case match_data[1]
when 'socket_path'
return -> { UNIXSocket.new(match_data[2]) }
when 'listen_addr'
addr, port = match_data[2].split(':')
return -> { TCPSocket.new(addr, port.to_i) }
end
end
raise "failed to find socket_path in #{config_path(service)}"
raise "failed to find socket_path or listen_addr in #{toml}"
end
def try_connect!(service)
def try_connect!(service, toml)
LOGGER.debug "Trying to connect to #{service}: "
timeout = 20
delay = 0.1
socket = read_socket_path(service)
connect = connect_proc(toml)
Integer(timeout / delay).times do
UNIXSocket.new(socket)
connect.call
LOGGER.debug " OK\n"
return
@ -194,6 +226,128 @@ module GitalySetup
LOGGER.warn " FAILED to connect to #{service}\n"
raise "could not connect to #{socket}"
raise "could not connect to #{service}"
end
def gitaly_socket_path
Gitlab::GitalyClient.address(REPOS_STORAGE).delete_prefix('unix:')
end
def gitaly_dir
socket_path = gitaly_socket_path
socket_path = File.expand_path(gitaly_socket_path) if expand_path_for_socket?
File.dirname(socket_path)
end
# Linux fails with "bind: invalid argument" if a UNIX socket path exceeds 108 characters:
# https://github.com/golang/go/issues/6895. We use absolute paths in CI to ensure
# that changes in the current working directory don't affect GRPC reconnections.
def expand_path_for_socket?
!!ENV['CI']
end
def setup_gitaly
unless ENV['CI']
# In CI Gitaly is built in the setup-test-env job and saved in the
# artifacts. So when tests are started, there's no need to build Gitaly.
build_gitaly
end
Gitlab::SetupHelper::Gitaly.create_configuration(
gitaly_dir,
{ 'default' => repos_path },
force: true,
options: {
prometheus_listen_addr: 'localhost:9236'
}
)
Gitlab::SetupHelper::Gitaly.create_configuration(
gitaly_dir,
{ 'default' => repos_path },
force: true,
options: {
internal_socket_dir: File.join(gitaly_dir, "internal_gitaly2"),
gitaly_socket: "gitaly2.socket",
config_filename: "gitaly2.config.toml"
}
)
Gitlab::SetupHelper::Praefect.create_configuration(gitaly_dir, { 'praefect' => repos_path }, force: true)
end
def socket_path(service)
File.join(tmp_tests_gitaly_dir, "#{service}.socket")
end
def praefect_socket_path
"unix:" + socket_path(:praefect)
end
def stop(pid)
Process.kill('KILL', pid)
rescue Errno::ESRCH
# The process can already be gone if the test run was INTerrupted.
end
def spawn_gitaly(toml = nil)
check_gitaly_config!
pids = []
if toml
pids << start_gitaly(toml)
else
pids << start_gitaly
pids << start_gitaly2
pids << start_praefect
end
Kernel.at_exit do
# In CI, this function is called by scripts/gitaly-test-spawn, triggered
# in a before_script. Gitaly needs to remain running until the container
# is stopped.
next if ENV['CI']
# In Workhorse tests (locally or in CI), this function is called by
# scripts/gitaly-test-spawn during `make test`. Gitaly needs to remain
# running until `make test` cleans it up.
next if ENV['GITALY_PID_FILE']
pids.each { |pid| stop(pid) }
end
rescue StandardError
raise gitaly_failure_message
end
def gitaly_failure_message
message = "gitaly spawn failed\n\n"
message += "- The `gitaly` binary does not exist: #{gitaly_binary}\n" unless File.exist?(gitaly_binary)
message += "- The `praefect` binary does not exist: #{praefect_binary}\n" unless File.exist?(praefect_binary)
message += "- The `git` binary does not exist: #{git_binary}\n" unless File.exist?(git_binary)
message += "\nCheck log/gitaly-test.log for errors.\n"
unless ci?
message += "\nIf binaries are missing, try running `make -C tmp/tests/gitaly build git.`\n"
message += "\nOtherwise, try running `rm -rf #{tmp_tests_gitaly_dir}`."
end
message
end
def git_binary
File.join(tmp_tests_gitaly_dir, "_build", "deps", "git", "install", "bin", "git")
end
def gitaly_binary
File.join(tmp_tests_gitaly_dir, "_build", "bin", "gitaly")
end
def praefect_binary
File.join(tmp_tests_gitaly_dir, "_build", "bin", "praefect")
end
def git_binary_exists?
File.exist?(git_binary)
end
end

View File

@ -1,6 +1,7 @@
# frozen_string_literal: true
require 'parallel'
require_relative 'gitaly_setup'
module TestEnv
extend self
@ -93,7 +94,6 @@ module TestEnv
}.freeze
TMP_TEST_PATH = Rails.root.join('tmp', 'tests').freeze
REPOS_STORAGE = 'default'
SECOND_STORAGE_PATH = Rails.root.join('tmp', 'tests', 'second_storage')
SETUP_METHODS = %i[setup_gitaly setup_gitlab_shell setup_workhorse setup_factory_repo setup_forked_repo].freeze
@ -128,7 +128,7 @@ module TestEnv
# Can be overriden
def post_init
start_gitaly(gitaly_dir)
start_gitaly
end
# Clean /tmp/tests
@ -142,7 +142,7 @@ module TestEnv
end
FileUtils.mkdir_p(
Gitlab::GitalyClient::StorageSettings.allow_disk_access { TestEnv.repos_path }
Gitlab::GitalyClient::StorageSettings.allow_disk_access { GitalySetup.repos_path }
)
FileUtils.mkdir_p(SECOND_STORAGE_PATH)
FileUtils.mkdir_p(backup_path)
@ -159,109 +159,28 @@ module TestEnv
def setup_gitaly
component_timed_setup('Gitaly',
install_dir: gitaly_dir,
install_dir: GitalySetup.gitaly_dir,
version: Gitlab::GitalyClient.expected_server_version,
task: "gitlab:gitaly:test_install",
task_args: [gitaly_dir, repos_path, gitaly_url].compact) do
Gitlab::SetupHelper::Gitaly.create_configuration(
gitaly_dir,
{ 'default' => repos_path },
force: true,
options: {
prometheus_listen_addr: 'localhost:9236'
}
)
Gitlab::SetupHelper::Gitaly.create_configuration(
gitaly_dir,
{ 'default' => repos_path },
force: true,
options: {
internal_socket_dir: File.join(gitaly_dir, "internal_gitaly2"),
gitaly_socket: "gitaly2.socket",
config_filename: "gitaly2.config.toml"
}
)
Gitlab::SetupHelper::Praefect.create_configuration(gitaly_dir, { 'praefect' => repos_path }, force: true)
end
task: "gitlab:gitaly:clone",
fresh_install: ENV.key?('FORCE_GITALY_INSTALL'),
task_args: [GitalySetup.gitaly_dir, GitalySetup.repos_path, gitaly_url].compact) do
GitalySetup.setup_gitaly
end
end
def gitaly_socket_path
Gitlab::GitalyClient.address('default').sub(/\Aunix:/, '')
end
def gitaly_dir
socket_path = gitaly_socket_path
socket_path = File.expand_path(gitaly_socket_path) if expand_path?
File.dirname(socket_path)
end
# Linux fails with "bind: invalid argument" if a UNIX socket path exceeds 108 characters:
# https://github.com/golang/go/issues/6895. We use absolute paths in CI to ensure
# that changes in the current working directory don't affect GRPC reconnections.
def expand_path?
!!ENV['CI']
end
def start_gitaly(gitaly_dir)
def start_gitaly
if ci?
# Gitaly has been spawned outside this process already
return
end
spawn_script = Rails.root.join('scripts/gitaly-test-spawn').to_s
Bundler.with_original_env do
unless system(spawn_script)
raise gitaly_failure_message
end
end
gitaly_pid = Integer(File.read(TMP_TEST_PATH.join('gitaly.pid')))
gitaly2_pid = Integer(File.read(TMP_TEST_PATH.join('gitaly2.pid')))
praefect_pid = Integer(File.read(TMP_TEST_PATH.join('praefect.pid')))
Kernel.at_exit do
pids = [gitaly_pid, gitaly2_pid, praefect_pid]
pids.each { |pid| stop(pid) }
end
wait('gitaly')
wait('praefect')
end
def stop(pid)
Process.kill('KILL', pid)
rescue Errno::ESRCH
# The process can already be gone if the test run was INTerrupted.
GitalySetup.spawn_gitaly
end
def gitaly_url
ENV.fetch('GITALY_REPO_URL', nil)
end
def socket_path(service)
TMP_TEST_PATH.join('gitaly', "#{service}.socket").to_s
end
def praefect_socket_path
"unix:" + socket_path(:praefect)
end
def wait(service)
sleep_time = 10
sleep_interval = 0.1
socket = socket_path(service)
Integer(sleep_time / sleep_interval).times do
Socket.unix(socket)
return
rescue StandardError
sleep sleep_interval
end
raise "could not connect to #{service} at #{socket.inspect} after #{sleep_time} seconds"
end
# Feature specs are run through Workhorse
def setup_workhorse
# Always rebuild the config file
@ -377,8 +296,7 @@ module TestEnv
def rm_storage_dir(storage, dir)
Gitlab::GitalyClient::StorageSettings.allow_disk_access do
repos_path = Gitlab.config.repositories.storages[storage].legacy_disk_path
target_repo_refs_path = File.join(repos_path, dir)
target_repo_refs_path = File.join(GitalySetup.repos_path(storage), dir)
FileUtils.remove_dir(target_repo_refs_path)
end
rescue Errno::ENOENT
@ -386,8 +304,7 @@ module TestEnv
def storage_dir_exists?(storage, dir)
Gitlab::GitalyClient::StorageSettings.allow_disk_access do
repos_path = Gitlab.config.repositories.storages[storage].legacy_disk_path
File.exist?(File.join(repos_path, dir))
File.exist?(File.join(GitalySetup.repos_path(storage), dir))
end
end
@ -400,7 +317,7 @@ module TestEnv
end
def repos_path
@repos_path ||= Gitlab.config.repositories.storages[REPOS_STORAGE].legacy_disk_path
@repos_path ||= GitalySetup.repos_path
end
def backup_path
@ -525,7 +442,7 @@ module TestEnv
end
end
def component_timed_setup(component, install_dir:, version:, task:, task_args: [])
def component_timed_setup(component, install_dir:, version:, task:, fresh_install: true, task_args: [])
start = Time.now
ensure_component_dir_name_is_correct!(component, install_dir)
@ -535,7 +452,7 @@ module TestEnv
if component_needs_update?(install_dir, version)
# Cleanup the component entirely to ensure we start fresh
FileUtils.rm_rf(install_dir)
FileUtils.rm_rf(install_dir) if fresh_install
if ENV['SKIP_RAILS_ENV_IN_RAKE']
# When we run `scripts/setup-test-env`, we take care of loading the necessary dependencies
@ -614,39 +531,6 @@ module TestEnv
expected_version == sha.chomp
end
def gitaly_failure_message
message = "gitaly spawn failed\n\n"
message += "- The `gitaly` binary does not exist: #{gitaly_binary}\n" unless File.exist?(gitaly_binary)
message += "- The `praefect` binary does not exist: #{praefect_binary}\n" unless File.exist?(praefect_binary)
message += "- The `git` binary does not exist: #{git_binary}\n" unless File.exist?(git_binary)
message += "\nCheck log/gitaly-test.log for errors.\n"
unless ci?
message += "\nIf binaries are missing, try running `make -C tmp/tests/gitaly build git.`\n"
message += "\nOtherwise, try running `rm -rf #{gitaly_dir}`."
end
message
end
def git_binary
File.join(gitaly_dir, "_build", "deps", "git", "install", "bin", "git")
end
def gitaly_binary
File.join(gitaly_dir, "_build", "bin", "gitaly")
end
def praefect_binary
File.join(gitaly_dir, "_build", "bin", "praefect")
end
def git_binary_exists?
File.exist?(git_binary)
end
end
require_relative('../../../ee/spec/support/helpers/ee/test_env') if Gitlab.ee?

View File

@ -1,11 +1,11 @@
# frozen_string_literal: true
require_relative 'helpers/test_env'
require_relative 'helpers/gitaly_setup'
RSpec.configure do |config|
config.before(:each, :praefect) do
allow(Gitlab.config.repositories.storages['default']).to receive(:[]).and_call_original
allow(Gitlab.config.repositories.storages['default']).to receive(:[]).with('gitaly_address')
.and_return(TestEnv.praefect_socket_path)
.and_return(GitalySetup.praefect_socket_path)
end
end

View File

@ -106,7 +106,7 @@ run-gitaly: $(GITALY_PID_FILE)
$(GITALY_PID_FILE): gitaly.toml
$(call message, "Starting gitaly")
cd ..; GITALY_TESTING_NO_GIT_HOOKS=1 GITALY_PID_FILE=workhorse/$(GITALY_PID_FILE) $(GITALY) workhorse/gitaly.toml &
cd ..; GITALY_TESTING_NO_GIT_HOOKS=1 GITALY_PID_FILE=workhorse/$(GITALY_PID_FILE) scripts/gitaly-test-spawn workhorse/gitaly.toml
gitaly.toml: ../tmp/tests/gitaly/config.toml
sed -e 's/^socket_path.*$$/listen_addr = "0.0.0.0:8075"/;s/^\[auth\]$$//;s/^token.*$$//;s/^internal_socket_dir.*$$//' \