From 3ba00693883c1003e4c3b9939ad1d3f1cba8592d Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Sat, 1 Oct 2022 03:08:44 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- .rubocop_todo/layout/line_length.yml | 2 +- .../development/ci_remove_userless_ci.yml | 8 --- .../geo_container_repository_replication.yml | 8 +++ .../monitoring/prometheus/gitlab_metrics.md | 4 ++ doc/api/geo_nodes.md | 32 ++++++---- doc/api/graphql/reference/index.md | 64 +++++++++++++++++++ lib/gitlab/git_access.rb | 5 +- locale/gitlab.pot | 3 + spec/lib/gitlab/git_access_spec.rb | 39 +---------- spec/requests/git_http_spec.rb | 63 +----------------- 10 files changed, 102 insertions(+), 126 deletions(-) delete mode 100644 config/feature_flags/development/ci_remove_userless_ci.yml create mode 100644 config/feature_flags/development/geo_container_repository_replication.yml diff --git a/.rubocop_todo/layout/line_length.yml b/.rubocop_todo/layout/line_length.yml index 9748ac81070..d115456b4fb 100644 --- a/.rubocop_todo/layout/line_length.yml +++ b/.rubocop_todo/layout/line_length.yml @@ -1877,7 +1877,7 @@ Layout/LineLength: - 'ee/spec/finders/ee/group_members_finder_spec.rb' - 'ee/spec/finders/ee/projects_finder_spec.rb' - 'ee/spec/finders/epics_finder_spec.rb' - - 'ee/spec/finders/geo/container_repository_registry_finder_spec.rb' + - 'ee/spec/finders/geo/container_repository_legacy_registry_finder_spec.rb' - 'ee/spec/finders/geo/project_registry_finder_spec.rb' - 'ee/spec/finders/geo/project_registry_status_finder_spec.rb' - 'ee/spec/finders/group_projects_finder_spec.rb' diff --git a/config/feature_flags/development/ci_remove_userless_ci.yml b/config/feature_flags/development/ci_remove_userless_ci.yml deleted file mode 100644 index 603ad14c6f0..00000000000 --- a/config/feature_flags/development/ci_remove_userless_ci.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -name: ci_remove_userless_ci -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/92942 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/368427 -milestone: '15.5' -type: development -group: group::pipeline execution -default_enabled: false diff --git a/config/feature_flags/development/geo_container_repository_replication.yml b/config/feature_flags/development/geo_container_repository_replication.yml new file mode 100644 index 00000000000..b849518a739 --- /dev/null +++ b/config/feature_flags/development/geo_container_repository_replication.yml @@ -0,0 +1,8 @@ +--- +name: geo_container_repository_replication +introduced_by_url: "https://gitlab.com/gitlab-org/gitlab/-/merge_requests/93690" +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/366662 +milestone: '15.5' +type: development +group: group::geo +default_enabled: false diff --git a/doc/administration/monitoring/prometheus/gitlab_metrics.md b/doc/administration/monitoring/prometheus/gitlab_metrics.md index 74bf15b2646..b3afd1892ec 100644 --- a/doc/administration/monitoring/prometheus/gitlab_metrics.md +++ b/doc/administration/monitoring/prometheus/gitlab_metrics.md @@ -302,6 +302,10 @@ configuration option in `gitlab.yml`. These metrics are served from the | `geo_uploads_verification_total` | Gauge | 14.6 | Number of uploads verifications tried on secondary | `url` | | `geo_uploads_verified` | Gauge | 14.6 | Number of uploads verified on secondary | `url` | | `geo_uploads_verification_failed` | Gauge | 14.6 | Number of uploads verifications failed on secondary | `url` | +| `geo_container_repositories` | Gauge | 15.4 | Number of container repositories on primary | `url` | +| `geo_container_repositories_synced` | Gauge | 15.4 | Number of container repositories synced on secondary | `url` | +| `geo_container_repositories_failed` | Gauge | 15.4 | Number of syncable container repositories failed to sync on secondary | `url` | +| `geo_container_repositories_registry` | Gauge | 15.4 | Number of container repositories in the registry | `url` | | `gitlab_sli:rails_request_apdex:total` | Counter | 14.4 | The number of request-apdex measurements, [more information the development documentation](../../../development/application_slis/rails_request_apdex.md) | `endpoint_id`, `feature_category`, `request_urgency` | | `gitlab_sli:rails_request_apdex:success_total` | Counter | 14.4 | The number of successful requests that met the target duration for their urgency. Divide by `gitlab_sli:rails_requests_apdex:total` to get a success ratio | `endpoint_id`, `feature_category`, `request_urgency` | | `geo_ci_secure_files` | Gauge | 15.3 | Number of secure files on primary | `url` | diff --git a/doc/api/geo_nodes.md b/doc/api/geo_nodes.md index c9fab6c0a2a..fe25b6661a0 100644 --- a/doc/api/geo_nodes.md +++ b/doc/api/geo_nodes.md @@ -339,10 +339,6 @@ Example response: "job_artifacts_failed_count": null, "job_artifacts_synced_missing_on_primary_count": 0, "job_artifacts_synced_in_percentage": "0.00%", - "container_repositories_count": 3, - "container_repositories_synced_count": null, - "container_repositories_failed_count": null, - "container_repositories_synced_in_percentage": "0.00%", "design_repositories_count": 3, "design_repositories_synced_count": null, "design_repositories_failed_count": null, @@ -507,7 +503,13 @@ Example response: "ci_secure_files_verification_failed_count": 0, "ci_secure_files_synced_in_percentage": "100.00%", "ci_secure_files_verified_in_percentage": "100.00%", - "ci_secure_files_synced_missing_on_primary_count": 0, + "ci_secure_files_synced_missing_on_primary_count": 0, + "container_repositories_count": 5, + "container_repositories_synced_count": 5, + "container_repositories_failed_count": 0, + "container_repositories_registry_count": 5, + "container_repositories_synced_in_percentage": "100.00%", + "container_repositories_synced_missing_on_primary_count": 0, }, { "geo_node_id": 2, @@ -533,10 +535,6 @@ Example response: "job_artifacts_failed_count": 1, "job_artifacts_synced_missing_on_primary_count": 0, "job_artifacts_synced_in_percentage": "50.00%", - "container_repositories_count": 3, - "container_repositories_synced_count": null, - "container_repositories_failed_count": null, - "container_repositories_synced_in_percentage": "0.00%", "design_repositories_count": 3, "design_repositories_synced_count": null, "design_repositories_failed_count": null, @@ -677,6 +675,12 @@ Example response: "job_artifacts_synced_in_percentage": "100.00%", "job_artifacts_verified_in_percentage": "100.00%", "job_artifacts_synced_missing_on_primary_count": 0, + "container_repositories_count": 5, + "container_repositories_synced_count": 5, + "container_repositories_failed_count": 0, + "container_repositories_registry_count": 5, + "container_repositories_synced_in_percentage": "100.00%", + "container_repositories_synced_missing_on_primary_count": 0, } ] ``` @@ -718,10 +722,6 @@ Example response: "job_artifacts_failed_count": 1, "job_artifacts_synced_missing_on_primary_count": 0, "job_artifacts_synced_in_percentage": "50.00%", - "container_repositories_count": 3, - "container_repositories_synced_count": null, - "container_repositories_failed_count": null, - "container_repositories_synced_in_percentage": "0.00%", "design_repositories_count": 3, "design_repositories_synced_count": null, "design_repositories_failed_count": null, @@ -856,6 +856,12 @@ Example response: "ci_secure_files_synced_in_percentage": "100.00%", "ci_secure_files_verified_in_percentage": "100.00%", "ci_secure_files_synced_missing_on_primary_count": 0, + "container_repositories_count": 5, + "container_repositories_synced_count": 5, + "container_repositories_failed_count": 0, + "container_repositories_registry_count": 5, + "container_repositories_synced_in_percentage": "100.00%", + "container_repositories_synced_missing_on_primary_count": 0, } ``` diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index a1564ed984d..e8318aed0d5 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -6816,6 +6816,29 @@ The edge type for [`ContainerRepository`](#containerrepository). | `cursor` | [`String!`](#string) | A cursor for use in pagination. | | `node` | [`ContainerRepository`](#containerrepository) | The item at the end of the edge. | +#### `ContainerRepositoryRegistryConnection` + +The connection type for [`ContainerRepositoryRegistry`](#containerrepositoryregistry). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `edges` | [`[ContainerRepositoryRegistryEdge]`](#containerrepositoryregistryedge) | A list of edges. | +| `nodes` | [`[ContainerRepositoryRegistry]`](#containerrepositoryregistry) | A list of nodes. | +| `pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | + +#### `ContainerRepositoryRegistryEdge` + +The edge type for [`ContainerRepositoryRegistry`](#containerrepositoryregistry). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `cursor` | [`String!`](#string) | A cursor for use in pagination. | +| `node` | [`ContainerRepositoryRegistry`](#containerrepositoryregistry) | The item at the end of the edge. | + #### `ContainerRepositoryTagConnection` The connection type for [`ContainerRepositoryTag`](#containerrepositorytag). @@ -10991,6 +11014,25 @@ four standard [pagination arguments](#connection-pagination-arguments): | `name` | [`String`](#string) | Search by tag name. | | `sort` | [`ContainerRepositoryTagSort`](#containerrepositorytagsort) | Sort tags by these criteria. | +### `ContainerRepositoryRegistry` + +Represents the Geo replication and verification state of an Container Repository. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `containerRepositoryId` | [`ID!`](#id) | ID of the ContainerRepository. | +| `createdAt` | [`Time`](#time) | Timestamp when the ContainerRepositoryRegistry was created. | +| `id` | [`ID!`](#id) | ID of the ContainerRepositoryRegistry. | +| `lastSyncFailure` | [`String`](#string) | Error message during sync of the ContainerRepositoryRegistry. | +| `lastSyncedAt` | [`Time`](#time) | Timestamp of the most recent successful sync of the ContainerRepositoryRegistry. | +| `retryAt` | [`Time`](#time) | Timestamp after which the ContainerRepositoryRegistry is resynced. | +| `retryCount` | [`Int`](#int) | Number of consecutive failed sync attempts of the ContainerRepositoryRegistry. | +| `state` | [`RegistryState`](#registrystate) | Sync state of the ContainerRepositoryRegistry. | +| `verificationRetryAt` | [`Time`](#time) | Timestamp after which the ContainerRepositoryRegistry is reverified. | +| `verifiedAt` | [`Time`](#time) | Timestamp of the most recent successful verification of the ContainerRepositoryRegistry. | + ### `ContainerRepositoryTag` A tag from a container repository. @@ -12368,6 +12410,28 @@ four standard [pagination arguments](#connection-pagination-arguments): | `replicationState` | [`ReplicationStateEnum`](#replicationstateenum) | Filters registries by their replication state. | | `verificationState` | [`VerificationStateEnum`](#verificationstateenum) | Filters registries by their verification state. | +##### `GeoNode.containerRepositoryRegistries` + +Find Container Repository registries on this Geo node. Ignored if `geo_container_repository_replication` feature flag is disabled. + +WARNING: +**Introduced** in 15.5. +This feature is in Alpha. It can be changed or removed at any time. + +Returns [`ContainerRepositoryRegistryConnection`](#containerrepositoryregistryconnection). + +This field returns a [connection](#connections). It accepts the +four standard [pagination arguments](#connection-pagination-arguments): +`before: String`, `after: String`, `first: Int`, `last: Int`. + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `ids` | [`[ID!]`](#id) | Filters registries by their ID. | +| `replicationState` | [`ReplicationStateEnum`](#replicationstateenum) | Filters registries by their replication state. | +| `verificationState` | [`VerificationStateEnum`](#verificationstateenum) | Filters registries by their verification state. | + ##### `GeoNode.groupWikiRepositoryRegistries` Find group wiki repository registries on this Geo node. diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 1d9f2d3265b..9a3f5fb844b 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -394,7 +394,7 @@ module Gitlab elsif user user.can?(:read_project, project) elsif ci? - !Feature.enabled?(:ci_remove_userless_ci, project) # When disabled allow CI (build without a user) for backward compatibility + false end || Guest.can?(:read_project, project) end @@ -445,9 +445,6 @@ module Gitlab nil when Key actor.user - when :ci - Gitlab::AppJsonLogger.info(message: 'Actor was :ci', project_id: project.id) - nil end end end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index edee00c8817..fd6310d4df0 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -10168,6 +10168,9 @@ msgstr "" msgid "Container Registry" msgstr "" +msgid "Container Repository" +msgstr "" + msgid "Container Scanning" msgstr "" diff --git a/spec/lib/gitlab/git_access_spec.rb b/spec/lib/gitlab/git_access_spec.rb index ac0c7c11aa0..cdebbe1539a 100644 --- a/spec/lib/gitlab/git_access_spec.rb +++ b/spec/lib/gitlab/git_access_spec.rb @@ -31,17 +31,6 @@ RSpec.describe Gitlab::GitAccess, :aggregate_failures do end end - shared_examples 'logs userless ci' do - it 'logs' do - expect(Gitlab::AppJsonLogger).to receive(:info).with( - message: 'Actor was :ci', - project_id: project.id - ).once - - pull_access_check - end - end - describe '#check with single protocols allowed' do def disable_protocol(protocol) allow(Gitlab::ProtocolAccess).to receive(:allowed?).with(protocol, project: project).and_return(false) @@ -149,7 +138,7 @@ RSpec.describe Gitlab::GitAccess, :aggregate_failures do end end - # For backwards compatibility + # legacy behavior that is blocked/deprecated context 'when actor is :ci' do let(:actor) { :ci } let(:authentication_abilities) { build_authentication_abilities } @@ -161,22 +150,6 @@ RSpec.describe Gitlab::GitAccess, :aggregate_failures do it 'does not block pushes with "not found"' do expect { push_access_check }.to raise_forbidden(described_class::ERROR_MESSAGES[:auth_upload]) end - - context 'when ci_remove_userless_ci is disabled' do - before do - stub_feature_flags(ci_remove_userless_ci: false) - end - - it 'allows pull access' do - expect { pull_access_check }.not_to raise_error - end - - it 'does not block pushes with "not found"' do - expect { push_access_check }.to raise_forbidden(described_class::ERROR_MESSAGES[:auth_upload]) - end - - it_behaves_like 'logs userless ci' - end end context 'when actor is DeployToken' do @@ -759,16 +732,6 @@ RSpec.describe Gitlab::GitAccess, :aggregate_failures do let(:actor) { :ci } specify { expect { pull_access_check }.to raise_error Gitlab::GitAccess::NotFoundError } - - context 'when ci_remove_userless_ci disabled' do - before do - stub_feature_flags(ci_remove_userless_ci: false) - end - - specify { expect { pull_access_check }.not_to raise_error } - - it_behaves_like 'logs userless ci' - end end end end diff --git a/spec/requests/git_http_spec.rb b/spec/requests/git_http_spec.rb index 7fadc75e2ad..5a774619e75 100644 --- a/spec/requests/git_http_spec.rb +++ b/spec/requests/git_http_spec.rb @@ -892,37 +892,6 @@ RSpec.describe 'Git HTTP requests' do expect(response).to have_gitlab_http_status(:forbidden) end - context 'when ci_remove_userless_ci disabled' do - before do - stub_feature_flags(ci_remove_userless_ci: false) - end - - it_behaves_like 'pulls are allowed' - - # A non-401 here is not an information leak since the system is - # "authenticated" as CI using the correct token. It does not have - # push access, so pushes should be rejected as forbidden, and giving - # a reason is fine. - # - # We know for sure it is not an information leak since pulls using - # the build token must be allowed. - it "rejects pushes with 403 Forbidden" do - push_get(path, **env) - - expect(response).to have_gitlab_http_status(:forbidden) - expect(response.body).to eq(git_access_error(:auth_upload)) - end - - # We are "authenticated" as CI using a valid token here. But we are - # not authorized to see any other project, so return "not found". - it "rejects pulls for other project with 404 Not Found" do - clone_get("#{other_project.full_path}.git", **env) - - expect(response).to have_gitlab_http_status(:not_found) - expect(response.body).to eq(git_access_error(:project_not_found)) - end - end - def pull download(path, **env) end @@ -1516,6 +1485,7 @@ RSpec.describe 'Git HTTP requests' do added_by: user) end + # legacy behavior that is blocked/deprecated context 'when build created by system is authenticated' do let(:path) { "#{project.full_path}.git" } let(:env) { { user: 'gitlab-ci-token', password: build.token } } @@ -1531,37 +1501,6 @@ RSpec.describe 'Git HTTP requests' do expect(response).to have_gitlab_http_status(:forbidden) end - - context 'when ci_remove_userless_ci is disabled' do - before do - stub_feature_flags(ci_remove_userless_ci: false) - end - - it_behaves_like 'pulls are allowed' - - # A non-401 here is not an information leak since the system is - # "authenticated" as CI using the correct token. It does not have - # push access, so pushes should be rejected as forbidden, and giving - # a reason is fine. - # - # We know for sure it is not an information leak since pulls using - # the build token must be allowed. - it "rejects pushes with 403 Forbidden" do - push_get(path, **env) - - expect(response).to have_gitlab_http_status(:forbidden) - expect(response.body).to eq(git_access_error(:auth_upload)) - end - - # We are "authenticated" as CI using a valid token here. But we are - # not authorized to see any other project, so return "not found". - it "rejects pulls for other project with 404 Not Found" do - clone_get("#{other_project.full_path}.git", **env) - - expect(response).to have_gitlab_http_status(:not_found) - expect(response.body).to eq(git_access_error(:project_not_found)) - end - end end context 'and build created by' do