diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml
index 1d65437ba12..830c6784a5f 100644
--- a/.gitlab/ci/rails.gitlab-ci.yml
+++ b/.gitlab/ci/rails.gitlab-ci.yml
@@ -463,7 +463,7 @@ db:backup_and_restore:
script:
- . scripts/prepare_build.sh
- bundle exec rake db:drop db:create db:structure:load db:seed_fu
- - mkdir -p tmp/tests/public/uploads tmp/tests/{artifacts,pages,lfs-objects,registry}
+ - mkdir -p tmp/tests/public/uploads tmp/tests/{artifacts,pages,lfs-objects,terraform_state,registry}
- bundle exec rake gitlab:backup:create
- date
- bundle exec rake gitlab:backup:restore
diff --git a/app/graphql/types/clusters/agent_token_type.rb b/app/graphql/types/clusters/agent_token_type.rb
index 94c5fc46a5d..ea3c02c52ba 100644
--- a/app/graphql/types/clusters/agent_token_type.rb
+++ b/app/graphql/types/clusters/agent_token_type.rb
@@ -44,6 +44,11 @@ module Types
null: true,
description: 'Name given to the token.'
+ field :status,
+ GraphQL::Types::String,
+ null: true,
+ description: 'Current status of the token.'
+
def cluster_agent
Gitlab::Graphql::Loaders::BatchModelLoader.new(::Clusters::Agent, object.agent_id).find
end
diff --git a/app/models/clusters/agent.rb b/app/models/clusters/agent.rb
index 98490a13351..8705c0fbec4 100644
--- a/app/models/clusters/agent.rb
+++ b/app/models/clusters/agent.rb
@@ -36,8 +36,8 @@ module Clusters
requested_project == project
end
- def active?
- agent_tokens.where("last_used_at > ?", INACTIVE_AFTER.ago).exists?
+ def connected?
+ agent_tokens.active.where("last_used_at > ?", INACTIVE_AFTER.ago).exists?
end
end
end
diff --git a/app/models/clusters/agent_token.rb b/app/models/clusters/agent_token.rb
index 87dba50cd69..d4f6beb045a 100644
--- a/app/models/clusters/agent_token.rb
+++ b/app/models/clusters/agent_token.rb
@@ -23,13 +23,18 @@ module Clusters
scope :order_last_used_at_desc, -> { order(::Gitlab::Database.nulls_last_order('last_used_at', 'DESC')) }
+ enum status: {
+ active: 0,
+ revoked: 1
+ }
+
def track_usage
track_values = { last_used_at: Time.current.utc }
cache_attributes(track_values)
if can_update_track_values?
- log_activity_event!(track_values[:last_used_at]) unless agent.active?
+ log_activity_event!(track_values[:last_used_at]) unless agent.connected?
# Use update_column so updated_at is skipped
update_columns(track_values)
diff --git a/db/fixtures/development/31_terraform_state.rb b/db/fixtures/development/31_terraform_state.rb
new file mode 100644
index 00000000000..76f9c270f72
--- /dev/null
+++ b/db/fixtures/development/31_terraform_state.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+TERRAFORM_FILE_VERSION = 1
+
+# Create sample terraform states in existing projects
+Gitlab::Seeder.quiet do
+ tfdata = {terraform_version: '0.14.1'}.to_json
+
+ Project.not_mass_generated.find_each do |project|
+ # Create as the project's creator
+ user = project.creator
+ # Set a build job source, if one exists for the project
+ build = project.builds.last
+
+ remote_state_handler = ::Terraform::RemoteStateHandler.new(project, user, name: project.path, lock_id: nil)
+
+ remote_state_handler.handle_with_lock do |state|
+ # Upload a file if a version does not already exist
+ state.update_file!(CarrierWaveStringFile.new(tfdata), version: TERRAFORM_FILE_VERSION, build: build) if state.latest_version.nil?
+ end
+
+ # rubocop:disable Rails/Output
+ print '.'
+ # rubocop:enable Rails/Output
+ end
+end
diff --git a/db/migrate/20211209230042_add_status_to_cluster_agent_tokens.rb b/db/migrate/20211209230042_add_status_to_cluster_agent_tokens.rb
new file mode 100644
index 00000000000..596c82eb209
--- /dev/null
+++ b/db/migrate/20211209230042_add_status_to_cluster_agent_tokens.rb
@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class AddStatusToClusterAgentTokens < Gitlab::Database::Migration[1.0]
+ def change
+ add_column :cluster_agent_tokens, :status, :smallint, null: false, default: 0
+ end
+end
diff --git a/db/post_migrate/20211123161906_cleanup_after_drop_invalid_security_findings.rb b/db/post_migrate/20211123161906_cleanup_after_drop_invalid_security_findings.rb
new file mode 100644
index 00000000000..599342e83e3
--- /dev/null
+++ b/db/post_migrate/20211123161906_cleanup_after_drop_invalid_security_findings.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+class CleanupAfterDropInvalidSecurityFindings < Gitlab::Database::Migration[1.0]
+ MIGRATION = "DropInvalidSecurityFindings"
+ INDEX_NAME = "tmp_index_uuid_is_null"
+
+ disable_ddl_transaction!
+
+ def up
+ # Make sure all jobs scheduled by
+ # db/post_migrate/20211110151350_schedule_drop_invalid_security_findings.rb
+ # are finished
+ finalize_background_migration(MIGRATION)
+ # Created by db/post_migrate/20211110151320_add_temporary_index_on_security_findings_uuid.rb
+ remove_concurrent_index_by_name :security_findings, INDEX_NAME
+ end
+
+ def down
+ add_concurrent_index(
+ :security_findings,
+ :id,
+ where: "uuid IS NULL",
+ name: INDEX_NAME
+ )
+ end
+end
diff --git a/db/schema_migrations/20211123161906 b/db/schema_migrations/20211123161906
new file mode 100644
index 00000000000..1370811b3af
--- /dev/null
+++ b/db/schema_migrations/20211123161906
@@ -0,0 +1 @@
+46767d804bde08ad4a076f20436652f980eb935a79b2ad30b4735b956be69a7a
\ No newline at end of file
diff --git a/db/schema_migrations/20211209230042 b/db/schema_migrations/20211209230042
new file mode 100644
index 00000000000..818734c1330
--- /dev/null
+++ b/db/schema_migrations/20211209230042
@@ -0,0 +1 @@
+907fafc18fa515fff8f716f6464263ccc8a9b6e5ead36f30b05089100fd71b6b
\ No newline at end of file
diff --git a/db/structure.sql b/db/structure.sql
index 1a8c1d2aeff..ba8edda0364 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -12384,6 +12384,7 @@ CREATE TABLE cluster_agent_tokens (
description text,
name text,
last_used_at timestamp with time zone,
+ status smallint DEFAULT 0 NOT NULL,
CONSTRAINT check_0fb634d04d CHECK ((name IS NOT NULL)),
CONSTRAINT check_2b79dbb315 CHECK ((char_length(name) <= 255)),
CONSTRAINT check_4e4ec5070a CHECK ((char_length(description) <= 1024)),
@@ -27982,8 +27983,6 @@ CREATE UNIQUE INDEX tmp_index_on_tmp_project_id_on_namespaces ON namespaces USIN
CREATE INDEX tmp_index_on_vulnerabilities_non_dismissed ON vulnerabilities USING btree (id) WHERE (state <> 2);
-CREATE INDEX tmp_index_uuid_is_null ON security_findings USING btree (id) WHERE (uuid IS NULL);
-
CREATE UNIQUE INDEX uniq_pkgs_deb_grp_architectures_on_distribution_id_and_name ON packages_debian_group_architectures USING btree (distribution_id, name);
CREATE UNIQUE INDEX uniq_pkgs_deb_grp_components_on_distribution_id_and_name ON packages_debian_group_components USING btree (distribution_id, name);
diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md
index 26526439273..a68a05536e8 100644
--- a/doc/api/graphql/reference/index.md
+++ b/doc/api/graphql/reference/index.md
@@ -8960,6 +8960,7 @@ GitLab CI/CD configuration template.
| `id` | [`ClustersAgentTokenID!`](#clustersagenttokenid) | Global ID of the token. |
| `lastUsedAt` | [`Time`](#time) | Timestamp the token was last used. |
| `name` | [`String`](#string) | Name given to the token. |
+| `status` | [`String`](#string) | Current status of the token. |
### `CodeCoverageActivity`
diff --git a/doc/raketasks/backup_restore.md b/doc/raketasks/backup_restore.md
index 676cc529c98..380541ab189 100644
--- a/doc/raketasks/backup_restore.md
+++ b/doc/raketasks/backup_restore.md
@@ -58,6 +58,7 @@ including:
- CI/CD job output logs
- CI/CD job artifacts
- LFS objects
+- Terraform states
- Container Registry images
- GitLab Pages content
- Snippets
@@ -65,7 +66,6 @@ including:
Backups do not include:
-- [Terraform state files](../administration/terraform_state.md)
- [Package registry files](../administration/packages/index.md)
- [Mattermost data](https://docs.mattermost.com/administration/config-settings.html#file-storage)
@@ -276,6 +276,7 @@ You can exclude specific directories from the backup by adding the environment v
- `builds` (CI job output logs)
- `artifacts` (CI job artifacts)
- `lfs` (LFS objects)
+- `terraform_state` (Terraform states)
- `registry` (Container Registry images)
- `pages` (Pages content)
- `repositories` (Git repositories data)
diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb
index 1bdc4965e5d..bbe18d52cf8 100644
--- a/lib/backup/manager.rb
+++ b/lib/backup/manager.rb
@@ -2,7 +2,7 @@
module Backup
class Manager
- ARCHIVES_TO_BACKUP = %w[uploads builds artifacts pages lfs registry].freeze
+ ARCHIVES_TO_BACKUP = %w[uploads builds artifacts pages lfs terraform_state registry].freeze
FOLDERS_TO_BACKUP = %w[repositories db].freeze
FILE_NAME_SUFFIX = '_gitlab_backup.tar'
diff --git a/lib/backup/terraform_state.rb b/lib/backup/terraform_state.rb
new file mode 100644
index 00000000000..5f71e18f1b4
--- /dev/null
+++ b/lib/backup/terraform_state.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Backup
+ class TerraformState < Backup::Files
+ attr_reader :progress
+
+ def initialize(progress)
+ @progress = progress
+
+ super('terraform_state', Settings.terraform_state.storage_path, excludes: ['tmp'])
+ end
+ end
+end
diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb
index 9c33a5fc872..ecda96af403 100644
--- a/lib/gitlab/auth/auth_finders.rb
+++ b/lib/gitlab/auth/auth_finders.rb
@@ -165,7 +165,7 @@ module Gitlab
authorization_token, _options = token_and_options(current_request)
- ::Clusters::AgentToken.find_by_token(authorization_token)
+ ::Clusters::AgentToken.active.find_by_token(authorization_token)
end
def find_runner_from_token
diff --git a/lib/gitlab/database/reindexing.rb b/lib/gitlab/database/reindexing.rb
index 6ffe14249f0..91c3fcc7d72 100644
--- a/lib/gitlab/database/reindexing.rb
+++ b/lib/gitlab/database/reindexing.rb
@@ -76,20 +76,7 @@ module Gitlab
def self.cleanup_leftovers!
PostgresIndex.reindexing_leftovers.each do |index|
- Gitlab::AppLogger.info("Removing index #{index.identifier} which is a leftover, temporary index from previous reindexing activity")
-
- retries = Gitlab::Database::WithLockRetriesOutsideTransaction.new(
- connection: index.connection,
- timing_configuration: REMOVE_INDEX_RETRY_CONFIG,
- klass: self.class,
- logger: Gitlab::AppLogger
- )
-
- retries.run(raise_on_exhaustion: false) do
- index.connection.tap do |conn|
- conn.execute("DROP INDEX CONCURRENTLY IF EXISTS #{conn.quote_table_name(index.schema)}.#{conn.quote_table_name(index.name)}")
- end
- end
+ Coordinator.new(index).drop
end
end
end
diff --git a/lib/gitlab/database/reindexing/coordinator.rb b/lib/gitlab/database/reindexing/coordinator.rb
index 3e4a83aa2e7..b4f7da999df 100644
--- a/lib/gitlab/database/reindexing/coordinator.rb
+++ b/lib/gitlab/database/reindexing/coordinator.rb
@@ -31,6 +31,25 @@ module Gitlab
end
end
+ def drop
+ try_obtain_lease do
+ Gitlab::AppLogger.info("Removing index #{index.identifier} which is a leftover, temporary index from previous reindexing activity")
+
+ retries = Gitlab::Database::WithLockRetriesOutsideTransaction.new(
+ connection: index.connection,
+ timing_configuration: REMOVE_INDEX_RETRY_CONFIG,
+ klass: self.class,
+ logger: Gitlab::AppLogger
+ )
+
+ retries.run(raise_on_exhaustion: false) do
+ index.connection.tap do |conn|
+ conn.execute("DROP INDEX CONCURRENTLY IF EXISTS #{conn.quote_table_name(index.schema)}.#{conn.quote_table_name(index.name)}")
+ end
+ end
+ end
+ end
+
private
def with_notifications(action)
diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake
index cc10d73f76a..f21b2caa58c 100644
--- a/lib/tasks/gitlab/backup.rake
+++ b/lib/tasks/gitlab/backup.rake
@@ -16,6 +16,7 @@ namespace :gitlab do
Rake::Task['gitlab:backup:artifacts:create'].invoke
Rake::Task['gitlab:backup:pages:create'].invoke
Rake::Task['gitlab:backup:lfs:create'].invoke
+ Rake::Task['gitlab:backup:terraform_state:create'].invoke
Rake::Task['gitlab:backup:registry:create'].invoke
backup = Backup::Manager.new(progress)
@@ -83,6 +84,7 @@ namespace :gitlab do
Rake::Task['gitlab:backup:artifacts:restore'].invoke unless backup.skipped?('artifacts')
Rake::Task['gitlab:backup:pages:restore'].invoke unless backup.skipped?('pages')
Rake::Task['gitlab:backup:lfs:restore'].invoke unless backup.skipped?('lfs')
+ Rake::Task['gitlab:backup:terraform_state:restore'].invoke unless backup.skipped?('terraform_state')
Rake::Task['gitlab:backup:registry:restore'].invoke unless backup.skipped?('registry')
Rake::Task['gitlab:shell:setup'].invoke
Rake::Task['cache:clear'].invoke
@@ -254,6 +256,25 @@ namespace :gitlab do
end
end
+ namespace :terraform_state do
+ task create: :gitlab_environment do
+ puts_time "Dumping terraform states ... ".color(:blue)
+
+ if ENV["SKIP"] && ENV["SKIP"].include?("terraform_state")
+ puts_time "[SKIPPED]".color(:cyan)
+ else
+ Backup::TerraformState.new(progress).dump
+ puts_time "done".color(:green)
+ end
+ end
+
+ task restore: :gitlab_environment do
+ puts_time "Restoring terraform states ... ".color(:blue)
+ Backup::TerraformState.new(progress).restore
+ puts_time "done".color(:green)
+ end
+ end
+
namespace :registry do
task create: :gitlab_environment do
puts_time "Dumping container registry images ... ".color(:blue)
diff --git a/shared/terraform_state/.gitkeep b/shared/terraform_state/.gitkeep
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/spec/graphql/types/clusters/agent_token_type_spec.rb b/spec/graphql/types/clusters/agent_token_type_spec.rb
index c872d201fd9..3f0720cb4b5 100644
--- a/spec/graphql/types/clusters/agent_token_type_spec.rb
+++ b/spec/graphql/types/clusters/agent_token_type_spec.rb
@@ -3,7 +3,7 @@
require 'spec_helper'
RSpec.describe GitlabSchema.types['ClusterAgentToken'] do
- let(:fields) { %i[cluster_agent created_at created_by_user description id last_used_at name] }
+ let(:fields) { %i[cluster_agent created_at created_by_user description id last_used_at name status] }
it { expect(described_class.graphql_name).to eq('ClusterAgentToken') }
diff --git a/spec/lib/backup/artifacts_spec.rb b/spec/lib/backup/artifacts_spec.rb
index 5a965030b01..102d787a5e1 100644
--- a/spec/lib/backup/artifacts_spec.rb
+++ b/spec/lib/backup/artifacts_spec.rb
@@ -12,7 +12,7 @@ RSpec.describe Backup::Artifacts do
Dir.mktmpdir do |tmpdir|
allow(JobArtifactUploader).to receive(:root) { "#{tmpdir}" }
- expect(backup.app_files_dir).to eq("#{tmpdir}")
+ expect(backup.app_files_dir).to eq("#{File.realpath(tmpdir)}")
end
end
end
diff --git a/spec/lib/backup/lfs_spec.rb b/spec/lib/backup/lfs_spec.rb
new file mode 100644
index 00000000000..fdc1c0c885d
--- /dev/null
+++ b/spec/lib/backup/lfs_spec.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Backup::Lfs do
+ let(:progress) { StringIO.new }
+
+ subject(:backup) { described_class.new(progress) }
+
+ describe '#dump' do
+ before do
+ allow(File).to receive(:realpath).and_call_original
+ allow(File).to receive(:realpath).with('/var/lfs-objects').and_return('/var/lfs-objects')
+ allow(File).to receive(:realpath).with('/var/lfs-objects/..').and_return('/var')
+ allow(Settings.lfs).to receive(:storage_path).and_return('/var/lfs-objects')
+ end
+
+ it 'uses the correct lfs dir in tar command', :aggregate_failures do
+ expect(backup.app_files_dir).to eq('/var/lfs-objects')
+ expect(backup).to receive(:tar).and_return('blabla-tar')
+ expect(backup).to receive(:run_pipeline!).with([%w(blabla-tar --exclude=lost+found -C /var/lfs-objects -cf - .), 'gzip -c -1'], any_args).and_return([[true, true], ''])
+ expect(backup).to receive(:pipeline_succeeded?).and_return(true)
+
+ backup.dump
+ end
+ end
+end
diff --git a/spec/lib/backup/manager_spec.rb b/spec/lib/backup/manager_spec.rb
index 32eea82cfdf..0e26e2faa5c 100644
--- a/spec/lib/backup/manager_spec.rb
+++ b/spec/lib/backup/manager_spec.rb
@@ -15,7 +15,7 @@ RSpec.describe Backup::Manager do
end
describe '#pack' do
- let(:expected_backup_contents) { %w(repositories db uploads.tar.gz builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz backup_information.yml) }
+ let(:expected_backup_contents) { %w(repositories db uploads.tar.gz builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz terraform_state.tar.gz backup_information.yml) }
let(:tar_file) { '1546300800_2019_01_01_12.3_gitlab_backup.tar' }
let(:tar_system_options) { { out: [tar_file, 'w', Gitlab.config.backup.archive_permissions] } }
let(:tar_cmdline) { ['tar', '-cf', '-', *expected_backup_contents, tar_system_options] }
@@ -57,7 +57,7 @@ RSpec.describe Backup::Manager do
end
context 'when skipped is set in backup_information.yml' do
- let(:expected_backup_contents) { %w{db uploads.tar.gz builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz backup_information.yml} }
+ let(:expected_backup_contents) { %w{db uploads.tar.gz builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz terraform_state.tar.gz backup_information.yml} }
let(:backup_information) do
{
backup_created_at: Time.zone.parse('2019-01-01'),
@@ -74,7 +74,7 @@ RSpec.describe Backup::Manager do
end
context 'when a directory does not exist' do
- let(:expected_backup_contents) { %w{db uploads.tar.gz builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz backup_information.yml} }
+ let(:expected_backup_contents) { %w{db uploads.tar.gz builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz terraform_state.tar.gz backup_information.yml} }
before do
expect(Dir).to receive(:exist?).with(File.join(Gitlab.config.backup.path, 'repositories')).and_return(false)
diff --git a/spec/lib/backup/terraform_state_spec.rb b/spec/lib/backup/terraform_state_spec.rb
new file mode 100644
index 00000000000..56051501204
--- /dev/null
+++ b/spec/lib/backup/terraform_state_spec.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Backup::TerraformState do
+ let(:progress) { StringIO.new }
+
+ subject(:backup) { described_class.new(progress) }
+
+ describe '#dump' do
+ before do
+ allow(File).to receive(:realpath).and_call_original
+ allow(File).to receive(:realpath).with('/var/terraform_state').and_return('/var/terraform_state')
+ allow(File).to receive(:realpath).with('/var/terraform_state/..').and_return('/var')
+ allow(Settings.terraform_state).to receive(:storage_path).and_return('/var/terraform_state')
+ end
+
+ it 'uses the correct storage dir in tar command and excludes tmp', :aggregate_failures do
+ expect(backup.app_files_dir).to eq('/var/terraform_state')
+ expect(backup).to receive(:tar).and_return('blabla-tar')
+ expect(backup).to receive(:run_pipeline!).with([%w(blabla-tar --exclude=lost+found --exclude=./tmp -C /var/terraform_state -cf - .), 'gzip -c -1'], any_args).and_return([[true, true], ''])
+ expect(backup).to receive(:pipeline_succeeded?).and_return(true)
+
+ backup.dump
+ end
+ end
+end
diff --git a/spec/lib/backup/uploads_spec.rb b/spec/lib/backup/uploads_spec.rb
index a82cb764f4d..c173916fe91 100644
--- a/spec/lib/backup/uploads_spec.rb
+++ b/spec/lib/backup/uploads_spec.rb
@@ -14,13 +14,14 @@ RSpec.describe Backup::Uploads do
allow(Gitlab.config.uploads).to receive(:storage_path) { tmpdir }
- expect(backup.app_files_dir).to eq("#{tmpdir}/uploads")
+ expect(backup.app_files_dir).to eq("#{File.realpath(tmpdir)}/uploads")
end
end
end
describe '#dump' do
before do
+ allow(File).to receive(:realpath).and_call_original
allow(File).to receive(:realpath).with('/var/uploads').and_return('/var/uploads')
allow(File).to receive(:realpath).with('/var/uploads/..').and_return('/var')
allow(Gitlab.config.uploads).to receive(:storage_path) { '/var' }
diff --git a/spec/lib/gitlab/auth/auth_finders_spec.rb b/spec/lib/gitlab/auth/auth_finders_spec.rb
index f1c891b2adb..e985f66bfe9 100644
--- a/spec/lib/gitlab/auth/auth_finders_spec.rb
+++ b/spec/lib/gitlab/auth/auth_finders_spec.rb
@@ -939,21 +939,19 @@ RSpec.describe Gitlab::Auth::AuthFinders do
end
describe '#cluster_agent_token_from_authorization_token' do
- let_it_be(:agent_token, freeze: true) { create(:cluster_agent_token) }
+ let_it_be(:agent_token) { create(:cluster_agent_token) }
+
+ subject { cluster_agent_token_from_authorization_token }
context 'when route_setting is empty' do
- it 'returns nil' do
- expect(cluster_agent_token_from_authorization_token).to be_nil
- end
+ it { is_expected.to be_nil }
end
context 'when route_setting allows cluster agent token' do
let(:route_authentication_setting) { { cluster_agent_token_allowed: true } }
context 'Authorization header is empty' do
- it 'returns nil' do
- expect(cluster_agent_token_from_authorization_token).to be_nil
- end
+ it { is_expected.to be_nil }
end
context 'Authorization header is incorrect' do
@@ -961,9 +959,7 @@ RSpec.describe Gitlab::Auth::AuthFinders do
request.headers['Authorization'] = 'Bearer ABCD'
end
- it 'returns nil' do
- expect(cluster_agent_token_from_authorization_token).to be_nil
- end
+ it { is_expected.to be_nil }
end
context 'Authorization header is malformed' do
@@ -971,9 +967,7 @@ RSpec.describe Gitlab::Auth::AuthFinders do
request.headers['Authorization'] = 'Bearer'
end
- it 'returns nil' do
- expect(cluster_agent_token_from_authorization_token).to be_nil
- end
+ it { is_expected.to be_nil }
end
context 'Authorization header matches agent token' do
@@ -981,8 +975,14 @@ RSpec.describe Gitlab::Auth::AuthFinders do
request.headers['Authorization'] = "Bearer #{agent_token.token}"
end
- it 'returns the agent token' do
- expect(cluster_agent_token_from_authorization_token).to eq(agent_token)
+ it { is_expected.to eq(agent_token) }
+
+ context 'agent token has been revoked' do
+ before do
+ agent_token.revoked!
+ end
+
+ it { is_expected.to be_nil }
end
end
end
diff --git a/spec/lib/gitlab/database/reindexing/coordinator_spec.rb b/spec/lib/gitlab/database/reindexing/coordinator_spec.rb
index 0afbe46b7f1..bb91617714a 100644
--- a/spec/lib/gitlab/database/reindexing/coordinator_spec.rb
+++ b/spec/lib/gitlab/database/reindexing/coordinator_spec.rb
@@ -6,30 +6,34 @@ RSpec.describe Gitlab::Database::Reindexing::Coordinator do
include Database::DatabaseHelpers
include ExclusiveLeaseHelpers
- describe '.perform' do
+ let(:notifier) { instance_double(Gitlab::Database::Reindexing::GrafanaNotifier, notify_start: nil, notify_end: nil) }
+ let(:index) { create(:postgres_index) }
+ let(:connection) { index.connection }
+
+ let!(:lease) { stub_exclusive_lease(lease_key, uuid, timeout: lease_timeout) }
+ let(:lease_key) { "gitlab/database/reindexing/coordinator/#{Gitlab::Database::PRIMARY_DATABASE_NAME}" }
+ let(:lease_timeout) { 1.day }
+ let(:uuid) { 'uuid' }
+
+ around do |example|
+ model = Gitlab::Database.database_base_models[Gitlab::Database::PRIMARY_DATABASE_NAME]
+
+ Gitlab::Database::SharedModel.using_connection(model.connection) do
+ example.run
+ end
+ end
+
+ before do
+ swapout_view_for_table(:postgres_indexes)
+ end
+
+ describe '#perform' do
subject { described_class.new(index, notifier).perform }
- let(:index) { create(:postgres_index) }
- let(:notifier) { instance_double(Gitlab::Database::Reindexing::GrafanaNotifier, notify_start: nil, notify_end: nil) }
let(:reindexer) { instance_double(Gitlab::Database::Reindexing::ReindexConcurrently, perform: nil) }
let(:action) { create(:reindex_action, index: index) }
- let!(:lease) { stub_exclusive_lease(lease_key, uuid, timeout: lease_timeout) }
- let(:lease_key) { "gitlab/database/reindexing/coordinator/#{Gitlab::Database::PRIMARY_DATABASE_NAME}" }
- let(:lease_timeout) { 1.day }
- let(:uuid) { 'uuid' }
-
- around do |example|
- model = Gitlab::Database.database_base_models[Gitlab::Database::PRIMARY_DATABASE_NAME]
-
- Gitlab::Database::SharedModel.using_connection(model.connection) do
- example.run
- end
- end
-
before do
- swapout_view_for_table(:postgres_indexes)
-
allow(Gitlab::Database::Reindexing::ReindexConcurrently).to receive(:new).with(index).and_return(reindexer)
allow(Gitlab::Database::Reindexing::ReindexAction).to receive(:create_for).with(index).and_return(action)
end
@@ -87,4 +91,40 @@ RSpec.describe Gitlab::Database::Reindexing::Coordinator do
end
end
end
+
+ describe '#drop' do
+ let(:connection) { index.connection }
+
+ subject(:drop) { described_class.new(index, notifier).drop }
+
+ context 'when exclusive lease is granted' do
+ it 'drops the index with lock retries' do
+ expect(lease).to receive(:try_obtain).ordered.and_return(uuid)
+
+ expect_query("SET lock_timeout TO '60000ms'")
+ expect_query("DROP INDEX CONCURRENTLY IF EXISTS \"public\".\"#{index.name}\"")
+ expect_query("RESET idle_in_transaction_session_timeout; RESET lock_timeout")
+
+ expect(Gitlab::ExclusiveLease).to receive(:cancel).ordered.with(lease_key, uuid)
+
+ drop
+ end
+
+ def expect_query(sql)
+ expect(connection).to receive(:execute).ordered.with(sql).and_wrap_original do |method, sql|
+ method.call(sql.sub(/CONCURRENTLY/, ''))
+ end
+ end
+ end
+
+ context 'when exclusive lease is not granted' do
+ it 'does not drop the index' do
+ expect(lease).to receive(:try_obtain).ordered.and_return(false)
+ expect(Gitlab::Database::WithLockRetriesOutsideTransaction).not_to receive(:new)
+ expect(connection).not_to receive(:execute)
+
+ drop
+ end
+ end
+ end
end
diff --git a/spec/models/clusters/agent_spec.rb b/spec/models/clusters/agent_spec.rb
index 3b521086c14..692831f6cca 100644
--- a/spec/models/clusters/agent_spec.rb
+++ b/spec/models/clusters/agent_spec.rb
@@ -76,12 +76,12 @@ RSpec.describe Clusters::Agent do
end
end
- describe '#active?' do
+ describe '#connected?' do
let_it_be(:agent) { create(:cluster_agent) }
let!(:token) { create(:cluster_agent_token, agent: agent, last_used_at: last_used_at) }
- subject { agent.active? }
+ subject { agent.connected? }
context 'agent has never connected' do
let(:last_used_at) { nil }
@@ -99,6 +99,14 @@ RSpec.describe Clusters::Agent do
let(:last_used_at) { 2.minutes.ago }
it { is_expected.to be_truthy }
+
+ context 'agent token has been revoked' do
+ before do
+ token.revoked!
+ end
+
+ it { is_expected.to be_falsey }
+ end
end
context 'agent has multiple tokens' do
diff --git a/spec/models/clusters/agent_token_spec.rb b/spec/models/clusters/agent_token_spec.rb
index ad9f948224f..462050b575a 100644
--- a/spec/models/clusters/agent_token_spec.rb
+++ b/spec/models/clusters/agent_token_spec.rb
@@ -9,6 +9,8 @@ RSpec.describe Clusters::AgentToken do
it { is_expected.to validate_length_of(:name).is_at_most(255) }
it { is_expected.to validate_presence_of(:name) }
+ it_behaves_like 'having unique enum values'
+
describe 'scopes' do
describe '.order_last_used_at_desc' do
let_it_be(:token_1) { create(:cluster_agent_token, last_used_at: 7.days.ago) }
@@ -76,9 +78,9 @@ RSpec.describe Clusters::AgentToken do
end
end
- context 'agent is inactive' do
+ context 'agent is not connected' do
before do
- allow(agent).to receive(:active?).and_return(false)
+ allow(agent).to receive(:connected?).and_return(false)
end
it 'creates an activity event' do
@@ -94,9 +96,9 @@ RSpec.describe Clusters::AgentToken do
end
end
- context 'agent is active' do
+ context 'agent is connected' do
before do
- allow(agent).to receive(:active?).and_return(true)
+ allow(agent).to receive(:connected?).and_return(true)
end
it 'does not create an activity event' do
diff --git a/spec/support/helpers/test_env.rb b/spec/support/helpers/test_env.rb
index d36bc4e3cb4..294e5bdd085 100644
--- a/spec/support/helpers/test_env.rb
+++ b/spec/support/helpers/test_env.rb
@@ -148,6 +148,8 @@ module TestEnv
FileUtils.mkdir_p(backup_path)
FileUtils.mkdir_p(pages_path)
FileUtils.mkdir_p(artifacts_path)
+ FileUtils.mkdir_p(lfs_path)
+ FileUtils.mkdir_p(terraform_state_path)
end
def setup_gitlab_shell
@@ -414,6 +416,14 @@ module TestEnv
Gitlab.config.artifacts.storage_path
end
+ def lfs_path
+ Gitlab.config.lfs.storage_path
+ end
+
+ def terraform_state_path
+ Gitlab.config.terraform_state.storage_path
+ end
+
# When no cached assets exist, manually hit the root path to create them
#
# Otherwise they'd be created by the first test, often timing out and
diff --git a/spec/tasks/gitlab/backup_rake_spec.rb b/spec/tasks/gitlab/backup_rake_spec.rb
index 99deaa8d154..5a7fbb42536 100644
--- a/spec/tasks/gitlab/backup_rake_spec.rb
+++ b/spec/tasks/gitlab/backup_rake_spec.rb
@@ -4,6 +4,7 @@ require 'rake_helper'
RSpec.describe 'gitlab:app namespace rake task', :delete do
let(:enable_registry) { true }
+ let(:backup_types) { %w{db repo uploads builds artifacts pages lfs terraform_state registry} }
def tars_glob
Dir.glob(File.join(Gitlab.config.backup.path, '*_gitlab_backup.tar'))
@@ -14,7 +15,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
end
def backup_files
- %w(backup_information.yml artifacts.tar.gz builds.tar.gz lfs.tar.gz pages.tar.gz)
+ %w(backup_information.yml artifacts.tar.gz builds.tar.gz lfs.tar.gz terraform_state.tar.gz pages.tar.gz)
end
def backup_directories
@@ -47,7 +48,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
end
def reenable_backup_sub_tasks
- %w{db repo uploads builds artifacts pages lfs registry}.each do |subtask|
+ backup_types.each do |subtask|
Rake::Task["gitlab:backup:#{subtask}:create"].reenable
end
end
@@ -71,14 +72,9 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
allow(YAML).to receive(:load_file)
.and_return({ gitlab_version: gitlab_version })
expect(Rake::Task['gitlab:db:drop_tables']).to receive(:invoke)
- expect(Rake::Task['gitlab:backup:db:restore']).to receive(:invoke)
- expect(Rake::Task['gitlab:backup:repo:restore']).to receive(:invoke)
- expect(Rake::Task['gitlab:backup:builds:restore']).to receive(:invoke)
- expect(Rake::Task['gitlab:backup:uploads:restore']).to receive(:invoke)
- expect(Rake::Task['gitlab:backup:artifacts:restore']).to receive(:invoke)
- expect(Rake::Task['gitlab:backup:pages:restore']).to receive(:invoke)
- expect(Rake::Task['gitlab:backup:lfs:restore']).to receive(:invoke)
- expect(Rake::Task['gitlab:backup:registry:restore']).to receive(:invoke)
+ backup_types.each do |subtask|
+ expect(Rake::Task["gitlab:backup:#{subtask}:restore"]).to receive(:invoke)
+ end
expect(Rake::Task['gitlab:shell:setup']).to receive(:invoke)
end
@@ -95,7 +91,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
context 'when the restore directory is not empty' do
before do
# We only need a backup of the repositories for this test
- stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,registry')
+ stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,terraform_state,registry')
create(:project, :repository)
end
@@ -139,11 +135,9 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
expect(Rake::Task['gitlab:backup:artifacts:restore']).to receive(:invoke)
expect(Rake::Task['gitlab:backup:pages:restore']).to receive(:invoke)
expect(Rake::Task['gitlab:backup:lfs:restore']).to receive(:invoke)
+ expect(Rake::Task['gitlab:backup:terraform_state:restore']).to receive(:invoke)
expect(Rake::Task['gitlab:backup:registry:restore']).to receive(:invoke)
expect(Rake::Task['gitlab:shell:setup']).to receive(:invoke)
-
- # We only need a backup of the repositories for this test
- stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,registry')
end
it 'restores the data' do
@@ -202,10 +196,8 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
end
context 'specific backup tasks' do
- let(:task_list) { %w(db repo uploads builds artifacts pages lfs registry) }
-
it 'prints a progress message to stdout' do
- task_list.each do |task|
+ backup_types.each do |task|
expect { run_rake_task("gitlab:backup:#{task}:create") }.to output(/Dumping /).to_stdout_from_any_process
end
end
@@ -219,10 +211,11 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping artifacts ... ")
expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping pages ... ")
expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping lfs objects ... ")
+ expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping terraform states ... ")
expect(Gitlab::BackupLogger).to receive(:info).with(message: "Dumping container registry images ... ")
- expect(Gitlab::BackupLogger).to receive(:info).with(message: "done").exactly(7).times
+ expect(Gitlab::BackupLogger).to receive(:info).with(message: "done").exactly(8).times
- task_list.each do |task|
+ backup_types.each do |task|
run_rake_task("gitlab:backup:#{task}:create")
end
end
@@ -255,7 +248,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
tar_contents, exit_status = Gitlab::Popen.popen(
- %W{tar -tvf #{backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz registry.tar.gz}
+ %W{tar -tvf #{backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz terraform_state.tar.gz registry.tar.gz}
)
expect(exit_status).to eq(0)
@@ -266,6 +259,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
expect(tar_contents).to match('artifacts.tar.gz')
expect(tar_contents).to match('pages.tar.gz')
expect(tar_contents).to match('lfs.tar.gz')
+ expect(tar_contents).to match('terraform_state.tar.gz')
expect(tar_contents).to match('registry.tar.gz')
expect(tar_contents).not_to match(%r{^.{4,9}[rwx].* (database.sql.gz|uploads.tar.gz|repositories|builds.tar.gz|pages.tar.gz|artifacts.tar.gz|registry.tar.gz)/$})
end
@@ -274,7 +268,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
temp_dirs = Dir.glob(
- File.join(Gitlab.config.backup.path, '{db,repositories,uploads,builds,artifacts,pages,lfs,registry}')
+ File.join(Gitlab.config.backup.path, '{db,repositories,uploads,builds,artifacts,pages,lfs,terraform_state,registry}')
)
expect(temp_dirs).to be_empty
@@ -304,7 +298,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
before do
# We only need a backup of the repositories for this test
- stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,registry')
+ stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,terraform_state,registry')
stub_storage_settings( second_storage_name => {
'gitaly_address' => Gitlab.config.repositories.storages.default.gitaly_address,
'path' => TestEnv::SECOND_STORAGE_PATH
@@ -378,7 +372,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
context 'concurrency settings' do
before do
# We only need a backup of the repositories for this test
- stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,registry')
+ stub_env('SKIP', 'db,uploads,builds,artifacts,lfs,terraform_state,registry')
create(:project, :repository)
end
@@ -425,31 +419,33 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
end
# backup_create task
- describe "Skipping items" do
+ describe "Skipping items in a backup" do
before do
- stub_env('SKIP', 'repositories,uploads')
+ stub_env('SKIP', 'an-unknown-type,repositories,uploads,anotherunknowntype')
create(:project, :repository)
end
- it "does not contain skipped item" do
+ it "does not contain repositories and uploads" do
expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
tar_contents, _exit_status = Gitlab::Popen.popen(
- %W{tar -tvf #{backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz registry.tar.gz}
+ %W{tar -tvf #{backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz pages.tar.gz lfs.tar.gz terraform_state.tar.gz registry.tar.gz}
)
expect(tar_contents).to match('db/')
- expect(tar_contents).to match('uploads.tar.gz')
+ expect(tar_contents).to match('uploads.tar.gz: Not found in archive')
expect(tar_contents).to match('builds.tar.gz')
expect(tar_contents).to match('artifacts.tar.gz')
expect(tar_contents).to match('lfs.tar.gz')
+ expect(tar_contents).to match('terraform_state.tar.gz')
expect(tar_contents).to match('pages.tar.gz')
expect(tar_contents).to match('registry.tar.gz')
expect(tar_contents).not_to match('repositories/')
+ expect(tar_contents).to match('repositories: Not found in archive')
end
- it 'does not invoke repositories restore' do
+ it 'does not invoke restore of repositories and uploads' do
expect { run_rake_task('gitlab:backup:create') }.to output.to_stdout_from_any_process
allow(Rake::Task['gitlab:shell:setup'])
@@ -463,6 +459,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
expect(Rake::Task['gitlab:backup:artifacts:restore']).to receive :invoke
expect(Rake::Task['gitlab:backup:pages:restore']).to receive :invoke
expect(Rake::Task['gitlab:backup:lfs:restore']).to receive :invoke
+ expect(Rake::Task['gitlab:backup:terraform_state:restore']).to receive :invoke
expect(Rake::Task['gitlab:backup:registry:restore']).to receive :invoke
expect(Rake::Task['gitlab:shell:setup']).to receive :invoke
expect { run_rake_task('gitlab:backup:restore') }.to output.to_stdout_from_any_process
@@ -488,6 +485,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
'builds.tar.gz',
'artifacts.tar.gz',
'lfs.tar.gz',
+ 'terraform_state.tar.gz',
'pages.tar.gz',
'registry.tar.gz',
'repositories'
@@ -501,14 +499,9 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do
.to receive(:invoke).and_return(true)
expect(Rake::Task['gitlab:db:drop_tables']).to receive :invoke
- expect(Rake::Task['gitlab:backup:db:restore']).to receive :invoke
- expect(Rake::Task['gitlab:backup:repo:restore']).to receive :invoke
- expect(Rake::Task['gitlab:backup:uploads:restore']).to receive :invoke
- expect(Rake::Task['gitlab:backup:builds:restore']).to receive :invoke
- expect(Rake::Task['gitlab:backup:artifacts:restore']).to receive :invoke
- expect(Rake::Task['gitlab:backup:pages:restore']).to receive :invoke
- expect(Rake::Task['gitlab:backup:lfs:restore']).to receive :invoke
- expect(Rake::Task['gitlab:backup:registry:restore']).to receive :invoke
+ backup_types.each do |subtask|
+ expect(Rake::Task["gitlab:backup:#{subtask}:restore"]).to receive :invoke
+ end
expect(Rake::Task['gitlab:shell:setup']).to receive :invoke
expect { run_rake_task("gitlab:backup:restore") }.to output.to_stdout_from_any_process
end