diff --git a/app/models/project_statistics.rb b/app/models/project_statistics.rb index 6d816932d39..7d7f9c2d5d6 100644 --- a/app/models/project_statistics.rb +++ b/app/models/project_statistics.rb @@ -106,8 +106,7 @@ class ProjectStatistics < ApplicationRecord packages_size + snippets_size + pipeline_artifacts_size + - uploads_size + - container_registry_size + uploads_size self.storage_size = storage_size end diff --git a/app/models/wiki.rb b/app/models/wiki.rb index b29c43bcfe1..78292c8e1fc 100644 --- a/app/models/wiki.rb +++ b/app/models/wiki.rb @@ -227,7 +227,7 @@ class Wiki end def create_page(title, content, format = :markdown, message = nil) - if Feature.enabled?(:gitaly_replace_wiki_create_page, container, default_enabled: :yaml) + if Feature.enabled?(:gitaly_replace_wiki_create_page, container, type: :undefined, default_enabled: :yaml) with_valid_format(format) do |default_extension| if file_exists_by_regex?(title) raise_duplicate_page_error! @@ -261,7 +261,7 @@ class Wiki end def update_page(page, content:, title: nil, format: :markdown, message: nil) - if Feature.enabled?(:gitaly_replace_wiki_update_page, container, default_enabled: :yaml) + if Feature.enabled?(:gitaly_replace_wiki_update_page, container, type: :undefined, default_enabled: :yaml) with_valid_format(format) do |default_extension| title = title.presence || Pathname(page.path).sub_ext('').to_s diff --git a/config/feature_flags/development/gitaly_enforce_requests_limits.yml b/config/feature_flags/undefined/gitaly_enforce_requests_limits.yml similarity index 87% rename from config/feature_flags/development/gitaly_enforce_requests_limits.yml rename to config/feature_flags/undefined/gitaly_enforce_requests_limits.yml index cacfd87f203..d624694d7b0 100644 --- a/config/feature_flags/development/gitaly_enforce_requests_limits.yml +++ b/config/feature_flags/undefined/gitaly_enforce_requests_limits.yml @@ -3,6 +3,6 @@ name: gitaly_enforce_requests_limits introduced_by_url: rollout_issue_url: milestone: -type: development +type: undefined group: default_enabled: false diff --git a/config/feature_flags/development/gitaly_go_user_merge_branch.yml b/config/feature_flags/undefined/gitaly_go_user_merge_branch.yml similarity index 87% rename from config/feature_flags/development/gitaly_go_user_merge_branch.yml rename to config/feature_flags/undefined/gitaly_go_user_merge_branch.yml index 3804a3f56b5..51b40b6aa38 100644 --- a/config/feature_flags/development/gitaly_go_user_merge_branch.yml +++ b/config/feature_flags/undefined/gitaly_go_user_merge_branch.yml @@ -3,6 +3,6 @@ name: gitaly_go_user_merge_branch introduced_by_url: rollout_issue_url: milestone: -type: development +type: undefined group: default_enabled: false diff --git a/config/feature_flags/development/gitaly_mep_mep.yml b/config/feature_flags/undefined/gitaly_mep_mep.yml similarity index 85% rename from config/feature_flags/development/gitaly_mep_mep.yml rename to config/feature_flags/undefined/gitaly_mep_mep.yml index 75cac13fe83..7f903b26539 100644 --- a/config/feature_flags/development/gitaly_mep_mep.yml +++ b/config/feature_flags/undefined/gitaly_mep_mep.yml @@ -3,6 +3,6 @@ name: gitaly_mep_mep introduced_by_url: rollout_issue_url: milestone: -type: development +type: undefined group: default_enabled: false diff --git a/config/feature_flags/development/gitaly_pack_objects_hook_with_sidechannel.yml b/config/feature_flags/undefined/gitaly_pack_objects_hook_with_sidechannel.yml similarity index 94% rename from config/feature_flags/development/gitaly_pack_objects_hook_with_sidechannel.yml rename to config/feature_flags/undefined/gitaly_pack_objects_hook_with_sidechannel.yml index 14cebc656c1..a09daecbfa5 100644 --- a/config/feature_flags/development/gitaly_pack_objects_hook_with_sidechannel.yml +++ b/config/feature_flags/undefined/gitaly_pack_objects_hook_with_sidechannel.yml @@ -3,6 +3,6 @@ name: gitaly_pack_objects_hook_with_sidechannel introduced_by_url: https://gitlab.com/gitlab-org/gitaly/-/merge_requests/3758 rollout_issue_url: https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1187 milestone: '14.3' -type: development +type: undefined group: 'team::Scalability' default_enabled: false diff --git a/config/feature_flags/development/gitaly_replace_wiki_create_page.yml b/config/feature_flags/undefined/gitaly_replace_wiki_create_page.yml similarity index 93% rename from config/feature_flags/development/gitaly_replace_wiki_create_page.yml rename to config/feature_flags/undefined/gitaly_replace_wiki_create_page.yml index 195bb71b5d2..9b7a36cb2c0 100644 --- a/config/feature_flags/development/gitaly_replace_wiki_create_page.yml +++ b/config/feature_flags/undefined/gitaly_replace_wiki_create_page.yml @@ -3,6 +3,6 @@ name: gitaly_replace_wiki_create_page introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/83734 rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/356983 milestone: '15.0' -type: development +type: undefined group: group::editor default_enabled: false diff --git a/config/feature_flags/development/gitaly_replace_wiki_update_page.yml b/config/feature_flags/undefined/gitaly_replace_wiki_update_page.yml similarity index 93% rename from config/feature_flags/development/gitaly_replace_wiki_update_page.yml rename to config/feature_flags/undefined/gitaly_replace_wiki_update_page.yml index 9fabf5edde8..4cc89769420 100644 --- a/config/feature_flags/development/gitaly_replace_wiki_update_page.yml +++ b/config/feature_flags/undefined/gitaly_replace_wiki_update_page.yml @@ -3,6 +3,6 @@ name: gitaly_replace_wiki_update_page introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/83833 rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/357246 milestone: '14.10' -type: development +type: undefined group: group::editor default_enabled: false diff --git a/config/feature_flags/development/gitaly_upload_pack_gitaly_hooks.yml b/config/feature_flags/undefined/gitaly_upload_pack_gitaly_hooks.yml similarity index 91% rename from config/feature_flags/development/gitaly_upload_pack_gitaly_hooks.yml rename to config/feature_flags/undefined/gitaly_upload_pack_gitaly_hooks.yml index 86e268af772..5d09ba62152 100644 --- a/config/feature_flags/development/gitaly_upload_pack_gitaly_hooks.yml +++ b/config/feature_flags/undefined/gitaly_upload_pack_gitaly_hooks.yml @@ -3,6 +3,6 @@ name: gitaly_upload_pack_gitaly_hooks introduced_by_url: rollout_issue_url: https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/807 milestone: '13.9' -type: development +type: undefined group: default_enabled: false diff --git a/config/feature_flags/development/gitaly_user_merge_branch_access_error.yml b/config/feature_flags/undefined/gitaly_user_merge_branch_access_error.yml similarity index 93% rename from config/feature_flags/development/gitaly_user_merge_branch_access_error.yml rename to config/feature_flags/undefined/gitaly_user_merge_branch_access_error.yml index 6e52112b754..2b732c3f1ec 100644 --- a/config/feature_flags/development/gitaly_user_merge_branch_access_error.yml +++ b/config/feature_flags/undefined/gitaly_user_merge_branch_access_error.yml @@ -3,6 +3,6 @@ name: gitaly_user_merge_branch_access_error introduced_by_url: https://gitlab.com/gitlab-org/gitaly/-/merge_requests/3705 rollout_issue_url: https://gitlab.com/gitlab-org/gitaly/-/issues/3757 milestone: '14.3' -type: development +type: undefined group: group::gitaly default_enabled: false diff --git a/data/deprecations/14-8-protect-cns-chs.yml b/data/deprecations/14-8-protect-cns-chs.yml index 8ea76d1b371..6fb67e98b2c 100644 --- a/data/deprecations/14-8-protect-cns-chs.yml +++ b/data/deprecations/14-8-protect-cns-chs.yml @@ -6,7 +6,7 @@ breaking_change: true reporter: sam.white body: | # Do not modify this line, instead modify the lines below. - All functionality related to GitLab's Container Network Security and Container Host Security categories is deprecated in GitLab 14.8 and scheduled for removal in GitLab 15.0. Users who need a replacement for this functionality are encouraged to evaluate the following open source projects as potential solutions that can be installed and managed outside of GitLab: [AppArmor](https://gitlab.com/apparmor/apparmor), [Cilium](https://github.com/cilium/cilium), [Falco](https://github.com/falcosecurity/falco), [FluentD](https://github.com/fluent/fluentd), [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/). To integrate these technologies into GitLab, add the desired Helm charts into your copy of the [Cluster Management Project Template](https://docs.gitlab.com/ee/user/clusters/management_project_template.html). Deploy these Helm charts in production by calling commands through the GitLab [Secure CI/CD Tunnel](https://docs.gitlab.com/ee/user/clusters/agent/repository.html#run-kubectl-commands-using-the-cicd-tunnel). + All functionality related to GitLab's Container Network Security and Container Host Security categories is deprecated in GitLab 14.8 and scheduled for removal in GitLab 15.0. Users who need a replacement for this functionality are encouraged to evaluate the following open source projects as potential solutions that can be installed and managed outside of GitLab: [AppArmor](https://gitlab.com/apparmor/apparmor), [Cilium](https://github.com/cilium/cilium), [Falco](https://github.com/falcosecurity/falco), [FluentD](https://github.com/fluent/fluentd), [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/). To integrate these technologies into GitLab, add the desired Helm charts into your copy of the [Cluster Management Project Template](https://docs.gitlab.com/ee/user/clusters/management_project_template.html). Deploy these Helm charts in production by calling commands through GitLab [CI/CD](https://docs.gitlab.com/ee/user/clusters/agent/ci_cd_workflow.html). As part of this change, the following specific capabilities within GitLab are now deprecated, and are scheduled for removal in GitLab 15.0: diff --git a/data/removals/15_0/15-0-protect-cns-chs.yml b/data/removals/15_0/15-0-protect-cns-chs.yml index 0ace07ab408..470f809efa1 100644 --- a/data/removals/15_0/15-0-protect-cns-chs.yml +++ b/data/removals/15_0/15-0-protect-cns-chs.yml @@ -6,7 +6,7 @@ breaking_change: true reporter: sam.white body: | # Do not modify this line, instead modify the lines below. - All functionality related to the Container Network Security and Container Host Security categories was deprecated in GitLab 14.8 and is scheduled for removal in GitLab 15.0. Users who need a replacement for this functionality are encouraged to evaluate the following open source projects as potential solutions that can be installed and managed outside of GitLab: [AppArmor](https://gitlab.com/apparmor/apparmor), [Cilium](https://github.com/cilium/cilium), [Falco](https://github.com/falcosecurity/falco), [FluentD](https://github.com/fluent/fluentd), [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/). To integrate these technologies with GitLab, add the desired Helm charts in your copy of the [Cluster Management Project Template](https://docs.gitlab.com/ee/user/clusters/management_project_template.html). Deploy these Helm charts in production by calling commands through the GitLab [Secure CI/CD Tunnel](https://docs.gitlab.com/ee/user/clusters/agent/repository.html#run-kubectl-commands-using-the-cicd-tunnel). + All functionality related to the Container Network Security and Container Host Security categories was deprecated in GitLab 14.8 and is scheduled for removal in GitLab 15.0. Users who need a replacement for this functionality are encouraged to evaluate the following open source projects as potential solutions that can be installed and managed outside of GitLab: [AppArmor](https://gitlab.com/apparmor/apparmor), [Cilium](https://github.com/cilium/cilium), [Falco](https://github.com/falcosecurity/falco), [FluentD](https://github.com/fluent/fluentd), [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/). To integrate these technologies with GitLab, add the desired Helm charts in your copy of the [Cluster Management Project Template](https://docs.gitlab.com/ee/user/clusters/management_project_template.html). Deploy these Helm charts in production by calling commands through GitLab [CI/CD](https://docs.gitlab.com/ee/user/clusters/agent/ci_cd_workflow.html). As part of this change, the following capabilities within GitLab are scheduled for removal in GitLab 15.0: diff --git a/data/whats_new/202109200001_14_03.yml b/data/whats_new/202109200001_14_03.yml index ab82772e637..fa0f05bf0a0 100644 --- a/data/whats_new/202109200001_14_03.yml +++ b/data/whats_new/202109200001_14_03.yml @@ -72,7 +72,7 @@ release: 14.3 - title: Grant group access to the GitLab Kubernetes Agent body: | - The GitLab Kubernetes Agent provides a secure connection between a Kubernetes cluster and GitLab. Until GitLab 14.2, the CI/CD Tunnel enabled pushing to a cluster only from the same project where the Kubernetes Agent was registered. In GitLab 14.3, the Agent can be authorized to access entire groups. As a result, every project under the authorized group has access to the cluster without the need to register an agent for every project. + The GitLab Kubernetes Agent provides a secure connection between a Kubernetes cluster and GitLab. Until GitLab 14.2, the GitLab CI/CD enabled pushing to a cluster only from the same project where the Kubernetes Agent was registered. In GitLab 14.3, the Agent can be authorized to access entire groups. As a result, every project under the authorized group has access to the cluster without the need to register an agent for every project. stage: Configure self-managed: true gitlab-com: true diff --git a/data/whats_new/2021111800001_14_05.yml b/data/whats_new/2021111800001_14_05.yml index 5a7607478fc..71f4b0245b9 100644 --- a/data/whats_new/2021111800001_14_05.yml +++ b/data/whats_new/2021111800001_14_05.yml @@ -53,7 +53,7 @@ release: 14.5 - title: GitLab Kubernetes Agent available in GitLab Free body: | - Connecting a Kubernetes cluster with the GitLab Kubernetes Agent simplifies the setup for cluster applications and enables secure GitOps deployments to the cluster. Initially, the GitLab Kubernetes Agent was available only for Premium users. In our commitment to the open source ethos, we moved the core features of the GitLab Kubernetes Agent and the CI/CD Tunnel to GitLab Free. We expect that the open-sourced features are compelling to many users without dedicated infrastructure teams and strong requirements around cluster management. Advanced features remain available as part of the GitLab Premium offering. + Connecting a Kubernetes cluster with the GitLab Kubernetes Agent simplifies the setup for cluster applications and enables secure GitOps deployments to the cluster. Initially, the GitLab Kubernetes Agent was available only for Premium users. In our commitment to the open source ethos, we moved the core features of the GitLab agent for Kubernetes and the ability to use CI/CD with Kubernetes to GitLab Free. We expect that the open-sourced features are compelling to many users without dedicated infrastructure teams and strong requirements around cluster management. Advanced features remain available as part of the GitLab Premium offering. stage: Configure self-managed: true gitlab-com: true diff --git a/doc/administration/audit_events.md b/doc/administration/audit_events.md index 98fe05e2805..214a1f3a009 100644 --- a/doc/administration/audit_events.md +++ b/doc/administration/audit_events.md @@ -114,6 +114,7 @@ From there, you can see the following actions: - Instance administrator started or stopped impersonation of a group member. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/300961) in GitLab 14.8. - Group deploy token was successfully created, revoked, or deleted. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/353452) in GitLab 14.9. - Failed attempt to create a group deploy token. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/353452) in GitLab 14.9. +- [IP restrictions](../user/group/index.md#restrict-group-access-by-ip-address) changed. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/358986) in GitLab 15.0 Group events can also be accessed via the [Group Audit Events API](../api/audit_events.md#group-audit-events) diff --git a/doc/update/deprecations.md b/doc/update/deprecations.md index c69efe0ee81..a24b5ea8e3b 100644 --- a/doc/update/deprecations.md +++ b/doc/update/deprecations.md @@ -281,7 +281,7 @@ as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#brea Before updating GitLab, review the details carefully to determine if you need to make any changes to your code, settings, or workflow. -All functionality related to GitLab's Container Network Security and Container Host Security categories is deprecated in GitLab 14.8 and scheduled for removal in GitLab 15.0. Users who need a replacement for this functionality are encouraged to evaluate the following open source projects as potential solutions that can be installed and managed outside of GitLab: [AppArmor](https://gitlab.com/apparmor/apparmor), [Cilium](https://github.com/cilium/cilium), [Falco](https://github.com/falcosecurity/falco), [FluentD](https://github.com/fluent/fluentd), [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/). To integrate these technologies into GitLab, add the desired Helm charts into your copy of the [Cluster Management Project Template](https://docs.gitlab.com/ee/user/clusters/management_project_template.html). Deploy these Helm charts in production by calling commands through the GitLab [Secure CI/CD Tunnel](https://docs.gitlab.com/ee/user/clusters/agent/repository.html#run-kubectl-commands-using-the-cicd-tunnel). +All functionality related to GitLab's Container Network Security and Container Host Security categories is deprecated in GitLab 14.8 and scheduled for removal in GitLab 15.0. Users who need a replacement for this functionality are encouraged to evaluate the following open source projects as potential solutions that can be installed and managed outside of GitLab: [AppArmor](https://gitlab.com/apparmor/apparmor), [Cilium](https://github.com/cilium/cilium), [Falco](https://github.com/falcosecurity/falco), [FluentD](https://github.com/fluent/fluentd), [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/). To integrate these technologies into GitLab, add the desired Helm charts into your copy of the [Cluster Management Project Template](https://docs.gitlab.com/ee/user/clusters/management_project_template.html). Deploy these Helm charts in production by calling commands through GitLab [CI/CD](https://docs.gitlab.com/ee/user/clusters/agent/ci_cd_workflow.html). As part of this change, the following specific capabilities within GitLab are now deprecated, and are scheduled for removal in GitLab 15.0: diff --git a/doc/update/removals.md b/doc/update/removals.md index ffd8afabd47..88901d9af24 100644 --- a/doc/update/removals.md +++ b/doc/update/removals.md @@ -38,7 +38,7 @@ as a [breaking change](https://docs.gitlab.com/ee/development/contributing/#brea Before updating GitLab, review the details carefully to determine if you need to make any changes to your code, settings, or workflow. -All functionality related to the Container Network Security and Container Host Security categories was deprecated in GitLab 14.8 and is scheduled for removal in GitLab 15.0. Users who need a replacement for this functionality are encouraged to evaluate the following open source projects as potential solutions that can be installed and managed outside of GitLab: [AppArmor](https://gitlab.com/apparmor/apparmor), [Cilium](https://github.com/cilium/cilium), [Falco](https://github.com/falcosecurity/falco), [FluentD](https://github.com/fluent/fluentd), [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/). To integrate these technologies with GitLab, add the desired Helm charts in your copy of the [Cluster Management Project Template](https://docs.gitlab.com/ee/user/clusters/management_project_template.html). Deploy these Helm charts in production by calling commands through the GitLab [Secure CI/CD Tunnel](https://docs.gitlab.com/ee/user/clusters/agent/repository.html#run-kubectl-commands-using-the-cicd-tunnel). +All functionality related to the Container Network Security and Container Host Security categories was deprecated in GitLab 14.8 and is scheduled for removal in GitLab 15.0. Users who need a replacement for this functionality are encouraged to evaluate the following open source projects as potential solutions that can be installed and managed outside of GitLab: [AppArmor](https://gitlab.com/apparmor/apparmor), [Cilium](https://github.com/cilium/cilium), [Falco](https://github.com/falcosecurity/falco), [FluentD](https://github.com/fluent/fluentd), [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/). To integrate these technologies with GitLab, add the desired Helm charts in your copy of the [Cluster Management Project Template](https://docs.gitlab.com/ee/user/clusters/management_project_template.html). Deploy these Helm charts in production by calling commands through GitLab [CI/CD](https://docs.gitlab.com/ee/user/clusters/agent/ci_cd_workflow.html). As part of this change, the following capabilities within GitLab are scheduled for removal in GitLab 15.0: diff --git a/doc/user/clusters/agent/ci_cd_workflow.md b/doc/user/clusters/agent/ci_cd_workflow.md index b2944572faa..30fd8bc8b0d 100644 --- a/doc/user/clusters/agent/ci_cd_workflow.md +++ b/doc/user/clusters/agent/ci_cd_workflow.md @@ -4,7 +4,7 @@ group: Configure info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Using a GitLab CI/CD workflow for Kubernetes **(FREE)** +# Using GitLab CI/CD with a Kubernetes cluster **(FREE)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/327409) in GitLab 14.1. > - The pre-configured `KUBECONFIG` was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/324275) in GitLab 14.2. @@ -238,7 +238,7 @@ See the [official Kubernetes documentation for details](https://kubernetes.io/do ### `kubectl` commands not supported -The commands `kubectl exec`, `kubectl cp`, and `kubectl attach` are not supported. +The commands `kubectl exec`, `kubectl cp`, `kubectl attach`, and `kubectl port-forward` are not supported. Anything that uses these API endpoints does not work, because they use the deprecated SPDY protocol. [An issue exists](https://gitlab.com/gitlab-org/gitlab/-/issues/346248) to add support for these commands. diff --git a/doc/user/clusters/agent/gitops.md b/doc/user/clusters/agent/gitops.md index e99e3b00ec7..6ca9d855b44 100644 --- a/doc/user/clusters/agent/gitops.md +++ b/doc/user/clusters/agent/gitops.md @@ -4,7 +4,7 @@ group: Configure info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments --- -# Using a GitOps workflow for Kubernetes **(PREMIUM)** +# Using GitOps with a Kubernetes cluster **(PREMIUM)** > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/259669) in GitLab 13.7. > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/332227) in GitLab 14.0, the `resource_inclusions` and `resource_exclusions` attributes were removed and `reconcile_timeout`, `dry_run_strategy`, `prune`, `prune_timeout`, `prune_propagation_policy`, and `inventory_policy` attributes were added. diff --git a/lib/feature.rb b/lib/feature.rb index 8a911f9210d..32d3622399d 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -67,11 +67,10 @@ class Feature persisted_names.include?(feature_name.to_s) end - # use `default_enabled: true` to default the flag to being `enabled` - # unless set explicitly. The default is `disabled` - # TODO: remove the `default_enabled:` and read it from the `definition_yaml` - # check: https://gitlab.com/gitlab-org/gitlab/-/issues/30228 - def enabled?(key, thing = nil, type: :development, default_enabled: false) + # The default state of feature flag is read from `YAML` + # If feature flag does not have YAML it will fallback to `default_enabled: false` + # in production environment, but raise exception in development or tests + def enabled?(key, thing = nil, type: :development, default_enabled: :yaml) if check_feature_flags_definition? if thing && !thing.respond_to?(:flipper_id) raise InvalidFeatureFlagError, @@ -97,7 +96,7 @@ class Feature feature_value end - def disabled?(key, thing = nil, type: :development, default_enabled: false) + def disabled?(key, thing = nil, type: :development, default_enabled: :yaml) # we need to make different method calls to make it easy to mock / define expectations in test mode thing.nil? ? !enabled?(key, type: type, default_enabled: default_enabled) : !enabled?(key, thing, type: type, default_enabled: default_enabled) end diff --git a/lib/feature/gitaly.rb b/lib/feature/gitaly.rb index a1f7dc0ee39..610efe65057 100644 --- a/lib/feature/gitaly.rb +++ b/lib/feature/gitaly.rb @@ -8,7 +8,7 @@ class Feature def enabled?(feature_flag, project = nil) return false unless Feature::FlipperFeature.table_exists? - Feature.enabled?("#{PREFIX}#{feature_flag}", project) + Feature.enabled?("#{PREFIX}#{feature_flag}", project, type: :undefined, default_enabled: false) rescue ActiveRecord::NoDatabaseError, PG::ConnectionBad false end diff --git a/lib/feature/shared.rb b/lib/feature/shared.rb index 2ce078b2f02..40f21fc4f50 100644 --- a/lib/feature/shared.rb +++ b/lib/feature/shared.rb @@ -28,8 +28,8 @@ class Feature }, ops: { description: "Long-lived feature flags that control operational aspects of GitLab's behavior", - optional: true, - rollout_issue: false, + optional: false, + rollout_issue: true, ee_only: false, default_enabled: false, example: <<-EOS @@ -37,6 +37,14 @@ class Feature push_frontend_feature_flag(:my_ops_flag, project, type: :ops) EOS }, + undefined: { + description: "Feature flags that are undefined in GitLab codebase (should not be used)", + optional: true, + rollout_issue: false, + ee_only: false, + default_enabled: false, + example: '' + }, experiment: { description: 'Short lived, used specifically to run A/B/n experiments.', optional: true, diff --git a/lib/gitlab/doctor/secrets.rb b/lib/gitlab/doctor/secrets.rb index 44f5c97c70c..cd075569d10 100644 --- a/lib/gitlab/doctor/secrets.rb +++ b/lib/gitlab/doctor/secrets.rb @@ -30,14 +30,35 @@ module Gitlab private + # Skipping initializers may be needed if those attempt to access + # encrypted data on initialization and could fail because of it. + # + # format example: + # { + # model_class => { + # [ + # { action: :create, filters: [:before, :filter_name1] }, + # { action: :update, filters: [:after, :filter_name2] } + # ] + # } + # } + MODEL_INITIALIZERS_TO_SKIP = { + Integration => [ + { action: :initialize, filters: [:after, :initialize_properties] } + ] + }.freeze + def check_model_attributes(models_with_attributes) running_failures = 0 models_with_attributes.each do |model, attributes| failures_per_row = Hash.new { |h, k| h[k] = [] } - model.find_each do |data| - attributes.each do |att| - failures_per_row[data.id] << att unless valid_attribute?(data, att) + + with_skipped_callbacks_for(model) do + model.find_each do |data| + attributes.each do |att| + failures_per_row[data.id] << att unless valid_attribute?(data, att) + end end end @@ -82,6 +103,32 @@ module Gitlab false end + + # WARNING: using this logic in other places than a Rake task will need a + # different approach, as simply setting the callback again is not thread-safe + def with_skipped_callbacks_for(model) + raise StandardError, 'can only be used in a Rake environment' unless Gitlab::Runtime.rake? + + skip_callbacks_for_model(model) + + yield + + skip_callbacks_for_model(model, reset: true) + end + + def skip_callbacks_for_model(model, reset: false) + MODEL_INITIALIZERS_TO_SKIP.each do |klass, initializers| + next unless model <= klass + + initializers.each do |initializer| + if reset + model.set_callback(initializer[:action], *initializer[:filters]) + else + model.skip_callback(initializer[:action], *initializer[:filters]) + end + end + end + end end end end diff --git a/lib/gitlab/metrics/methods.rb b/lib/gitlab/metrics/methods.rb index dc9a7ed1312..78b00aa04fe 100644 --- a/lib/gitlab/metrics/methods.rb +++ b/lib/gitlab/metrics/methods.rb @@ -56,7 +56,7 @@ module Gitlab end def disabled_by_feature(options) - options.with_feature && !::Feature.enabled?(options.with_feature, type: :ops) + options.with_feature && !::Feature.enabled?(options.with_feature, type: :undefined, default_enabled: false) end def build_metric!(type, name, options) diff --git a/spec/lib/feature/definition_spec.rb b/spec/lib/feature/definition_spec.rb index 2f95f8eeab7..e5ada68effa 100644 --- a/spec/lib/feature/definition_spec.rb +++ b/spec/lib/feature/definition_spec.rb @@ -165,18 +165,14 @@ RSpec.describe Feature::Definition do using RSpec::Parameterized::TableSyntax let(:definition) do - Feature::Definition.new("development/enabled_feature_flag.yml", - name: :enabled_feature_flag, - type: 'development', - milestone: milestone, - default_enabled: false) + described_class.new("development/enabled_feature_flag.yml", + name: :enabled_feature_flag, + type: 'development', + milestone: milestone, + default_enabled: false) end before do - allow(Feature::Definition).to receive(:definitions) do - { definition.key => definition } - end - allow(Gitlab).to receive(:version_info).and_return(Gitlab::VersionInfo.parse(current_milestone)) end @@ -192,7 +188,7 @@ RSpec.describe Feature::Definition do end with_them do - it {is_expected.to be(expected)} + it { is_expected.to be(expected) } end end @@ -254,23 +250,23 @@ RSpec.describe Feature::Definition do using RSpec::Parameterized::TableSyntax let(:definition) do - Feature::Definition.new("development/enabled_feature_flag.yml", - name: :enabled_feature_flag, - type: 'development', - milestone: milestone, - log_state_changes: log_state_change, - default_enabled: false) + described_class.new("development/enabled_feature_flag.yml", + name: :enabled_feature_flag, + type: 'development', + milestone: milestone, + log_state_changes: log_state_change, + default_enabled: false) end before do - allow(Feature::Definition).to receive(:definitions) do - { definition.key => definition } - end + stub_feature_flag_definition(:enabled_feature_flag, + milestone: milestone, + log_state_changes: log_state_change) allow(Gitlab).to receive(:version_info).and_return(Gitlab::VersionInfo.new(10, 0, 0)) end - subject { Feature::Definition.log_states?(key) } + subject { described_class.log_states?(key) } where(:ctx, :key, :milestone, :log_state_change, :expected) do 'When flag does not exist' | :no_flag | "0.0" | true | false diff --git a/spec/lib/feature_spec.rb b/spec/lib/feature_spec.rb index 54d95ff9e05..92ad899fc5e 100644 --- a/spec/lib/feature_spec.rb +++ b/spec/lib/feature_spec.rb @@ -157,6 +157,9 @@ RSpec.describe Feature, stub_feature_flags: false do describe '.enabled?' do before do allow(Feature).to receive(:log_feature_flag_states?).and_return(false) + + stub_feature_flag_definition(:disabled_feature_flag) + stub_feature_flag_definition(:enabled_feature_flag) end context 'when self-recursive' do @@ -170,15 +173,15 @@ RSpec.describe Feature, stub_feature_flags: false do end it 'returns the default value' do - expect(described_class.enabled?(:ff, default_enabled: true)).to eq true + expect(described_class.enabled?(:enabled_feature_flag, default_enabled: true)).to eq true end it 'detects self recursion' do expect(Gitlab::ErrorTracking) .to receive(:track_exception) - .with(have_attributes(message: 'self recursion'), { stack: [:ff] }) + .with(have_attributes(message: 'self recursion'), { stack: [:enabled_feature_flag] }) - described_class.enabled?(:ff) + described_class.enabled?(:enabled_feature_flag) end end @@ -186,7 +189,7 @@ RSpec.describe Feature, stub_feature_flags: false do before do allow(Feature).to receive(:with_feature).and_wrap_original do |original, name, &block| original.call(name) do |ff| - Feature.enabled?(:"deeper_#{name}") + Feature.enabled?(:"deeper_#{name}", type: :undefined, default_enabled: true) block.call(ff) end end @@ -197,15 +200,21 @@ RSpec.describe Feature, stub_feature_flags: false do .to receive(:track_exception) .with(have_attributes(message: 'deep recursion'), stack: have_attributes(size: be > 10)) - described_class.enabled?(:ff) + described_class.enabled?(:enabled_feature_flag) end end - it 'returns false for undefined feature' do + it 'returns false (and tracks / raises exception for dev) for undefined feature' do + expect(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception) + expect(described_class.enabled?(:some_random_feature_flag)).to be_falsey end - it 'returns true for undefined feature with default_enabled' do + it 'returns false for undefined feature with default_enabled: false' do + expect(described_class.enabled?(:some_random_feature_flag, default_enabled: false)).to be_falsey + end + + it 'returns true for undefined feature with default_enabled: true' do expect(described_class.enabled?(:some_random_feature_flag, default_enabled: true)).to be_truthy end @@ -257,15 +266,7 @@ RSpec.describe Feature, stub_feature_flags: false do before do allow(Feature).to receive(:log_feature_flag_states?).and_call_original - definition = Feature::Definition.new("development/enabled_feature_flag.yml", - name: :enabled_feature_flag, - type: 'development', - log_state_changes: true, - default_enabled: false) - - allow(Feature::Definition).to receive(:definitions) do - { definition.key => definition } - end + stub_feature_flag_definition(:enabled_feature_flag, log_state_changes: true) described_class.enable(:feature_flag_state_logs) described_class.enable(:enabled_feature_flag) @@ -283,7 +284,7 @@ RSpec.describe Feature, stub_feature_flags: false do end context 'cached feature flag', :request_store do - let(:flag) { :some_feature_flag } + let(:flag) { :enabled_feature_flag } before do described_class.send(:flipper).memoize = false @@ -467,21 +468,29 @@ RSpec.describe Feature, stub_feature_flags: false do end describe '.disable?' do - it 'returns true for undefined feature' do + it 'returns true (and tracks / raises exception for dev) for undefined feature' do + expect(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception) + expect(described_class.disabled?(:some_random_feature_flag)).to be_truthy end - it 'returns false for undefined feature with default_enabled' do + it 'returns true for undefined feature with default_enabled: false' do + expect(described_class.disabled?(:some_random_feature_flag, default_enabled: false)).to be_truthy + end + + it 'returns false for undefined feature with default_enabled: true' do expect(described_class.disabled?(:some_random_feature_flag, default_enabled: true)).to be_falsey end it 'returns true for existing disabled feature in the database' do + stub_feature_flag_definition(:disabled_feature_flag) described_class.disable(:disabled_feature_flag) expect(described_class.disabled?(:disabled_feature_flag)).to be_truthy end it 'returns false for existing enabled feature in the database' do + stub_feature_flag_definition(:enabled_feature_flag) described_class.enable(:enabled_feature_flag) expect(described_class.disabled?(:enabled_feature_flag)).to be_falsey @@ -620,14 +629,7 @@ RSpec.describe Feature, stub_feature_flags: false do let(:log_state_changes) { false } let(:milestone) { "0.0" } let(:flag_name) { :some_flag } - let(:definition) do - Feature::Definition.new("development/#{flag_name}.yml", - name: flag_name, - type: 'development', - milestone: milestone, - log_state_changes: log_state_changes, - default_enabled: false) - end + let(:flag_type) { 'development' } before do Feature.enable(:feature_flag_state_logs) @@ -637,9 +639,10 @@ RSpec.describe Feature, stub_feature_flags: false do allow(Feature).to receive(:log_feature_flag_states?).with(:feature_flag_state_logs).and_call_original allow(Feature).to receive(:log_feature_flag_states?).with(:some_flag).and_call_original - allow(Feature::Definition).to receive(:definitions) do - { definition.key => definition } - end + stub_feature_flag_definition(flag_name, + type: flag_type, + milestone: milestone, + log_state_changes: log_state_changes) end subject { described_class.log_feature_flag_states?(flag_name) } @@ -647,6 +650,7 @@ RSpec.describe Feature, stub_feature_flags: false do context 'when flag is feature_flag_state_logs' do let(:milestone) { "14.6" } let(:flag_name) { :feature_flag_state_logs } + let(:flag_type) { 'ops' } let(:log_state_changes) { true } it { is_expected.to be_falsey } @@ -657,13 +661,7 @@ RSpec.describe Feature, stub_feature_flags: false do end context 'when flag is old while log_state_changes is not present ' do - let(:definition) do - Feature::Definition.new("development/#{flag_name}.yml", - name: flag_name, - type: 'development', - milestone: milestone, - default_enabled: false) - end + let(:log_state_changes) { nil } it { is_expected.to be_falsey } end @@ -685,12 +683,7 @@ RSpec.describe Feature, stub_feature_flags: false do end context 'when milestone is nil' do - let(:definition) do - Feature::Definition.new("development/#{flag_name}.yml", - name: flag_name, - type: 'development', - default_enabled: false) - end + let(:milestone) { nil } it { is_expected.to be_falsey } end diff --git a/spec/lib/gitlab/doctor/secrets_spec.rb b/spec/lib/gitlab/doctor/secrets_spec.rb index f95a7eb1492..efdd6cc1199 100644 --- a/spec/lib/gitlab/doctor/secrets_spec.rb +++ b/spec/lib/gitlab/doctor/secrets_spec.rb @@ -7,10 +7,25 @@ RSpec.describe Gitlab::Doctor::Secrets do let!(:group) { create(:group, runners_token: "test") } let!(:project) { create(:project) } let!(:grafana_integration) { create(:grafana_integration, project: project, token: "test") } + let!(:integration) { create(:integration, project: project, properties: { test_key: "test_value" }) } let(:logger) { double(:logger).as_null_object } subject { described_class.new(logger).run! } + before do + allow(Gitlab::Runtime).to receive(:rake?).and_return(true) + end + + context 'when not ran in a Rake runtime' do + before do + allow(Gitlab::Runtime).to receive(:rake?).and_return(false) + end + + it 'raises an error' do + expect { subject }.to raise_error(StandardError, 'can only be used in a Rake environment') + end + end + context 'when encrypted attributes are properly set' do it 'detects decryptable secrets' do expect(logger).to receive(:info).with(/User failures: 0/) @@ -42,6 +57,25 @@ RSpec.describe Gitlab::Doctor::Secrets do end end + context 'when initializers attempt to use encrypted data' do + it 'skips the initializers and detects bad data' do + integration.encrypted_properties = "invalid" + integration.save! + + expect(logger).to receive(:info).with(/Integration failures: 1/) + + subject + end + + it 'resets the initializers after the task runs' do + subject + + expect(integration).to receive(:initialize_properties) + + integration.run_callbacks(:initialize) + end + end + context 'when GrafanaIntegration token is set via private method' do it 'can access GrafanaIntegration token value' do expect(logger).to receive(:info).with(/GrafanaIntegration failures: 0/) diff --git a/spec/lib/gitlab/gon_helper_spec.rb b/spec/lib/gitlab/gon_helper_spec.rb index 28cb9125af1..dd4dcca809b 100644 --- a/spec/lib/gitlab/gon_helper_spec.rb +++ b/spec/lib/gitlab/gon_helper_spec.rb @@ -44,6 +44,7 @@ RSpec.describe Gitlab::GonHelper do describe '#push_frontend_feature_flag' do before do skip_feature_flags_yaml_validation + skip_default_enabled_yaml_check end it 'pushes a feature flag to the frontend' do diff --git a/spec/lib/gitlab/lograge/custom_options_spec.rb b/spec/lib/gitlab/lograge/custom_options_spec.rb index d8f351bb8a3..58b05be6ff9 100644 --- a/spec/lib/gitlab/lograge/custom_options_spec.rb +++ b/spec/lib/gitlab/lograge/custom_options_spec.rb @@ -96,23 +96,15 @@ RSpec.describe Gitlab::Lograge::CustomOptions do end end - context 'when feature flags are present', :request_store do + context 'when feature flags are present', :request_store do before do allow(Feature).to receive(:log_feature_flag_states?).and_return(false) - definitions = {} [:enabled_feature, :disabled_feature].each do |flag_name| - definitions[flag_name] = Feature::Definition.new("development/enabled_feature.yml", - name: flag_name, - type: 'development', - log_state_changes: true, - default_enabled: false) - + stub_feature_flag_definition(flag_name, log_state_changes: true) allow(Feature).to receive(:log_feature_flag_states?).with(flag_name).and_call_original end - allow(Feature::Definition).to receive(:definitions).and_return(definitions) - Feature.enable(:enabled_feature) Feature.disable(:disabled_feature) end diff --git a/spec/models/project_statistics_spec.rb b/spec/models/project_statistics_spec.rb index 8244e0d052d..2c29d4c42f4 100644 --- a/spec/models/project_statistics_spec.rb +++ b/spec/models/project_statistics_spec.rb @@ -35,7 +35,7 @@ RSpec.describe ProjectStatistics do build_artifacts_size: 1.exabyte, snippets_size: 1.exabyte, pipeline_artifacts_size: 512.petabytes - 1, - uploads_size: 500.petabytes, + uploads_size: 512.petabytes, container_registry_size: 12.petabytes ) @@ -49,7 +49,7 @@ RSpec.describe ProjectStatistics do expect(statistics.storage_size).to eq(8.exabytes - 1) expect(statistics.snippets_size).to eq(1.exabyte) expect(statistics.pipeline_artifacts_size).to eq(512.petabytes - 1) - expect(statistics.uploads_size).to eq(500.petabytes) + expect(statistics.uploads_size).to eq(512.petabytes) expect(statistics.container_registry_size).to eq(12.petabytes) end end @@ -363,7 +363,7 @@ RSpec.describe ProjectStatistics do end describe '#update_storage_size' do - it "sums all storage counters" do + it "sums the relevant storage counters" do statistics.update!( repository_size: 2, wiki_size: 4, @@ -372,13 +372,24 @@ RSpec.describe ProjectStatistics do pipeline_artifacts_size: 3, build_artifacts_size: 3, packages_size: 6, + uploads_size: 5 + ) + + statistics.reload + + expect(statistics.storage_size).to eq 28 + end + + it 'excludes the container_registry_size' do + statistics.update!( + repository_size: 2, uploads_size: 5, container_registry_size: 10 ) statistics.reload - expect(statistics.storage_size).to eq 38 + expect(statistics.storage_size).to eq 7 end it 'works during wiki_size backfill' do diff --git a/spec/requests/api/features_spec.rb b/spec/requests/api/features_spec.rb index a265f67115a..6c88874dd5d 100644 --- a/spec/requests/api/features_spec.rb +++ b/spec/requests/api/features_spec.rb @@ -26,6 +26,7 @@ RSpec.describe API::Features, stub_feature_flags: false do end skip_feature_flags_yaml_validation + skip_default_enabled_yaml_check end describe 'GET /features' do diff --git a/spec/support/helpers/stub_feature_flags.rb b/spec/support/helpers/stub_feature_flags.rb index 77f31169ecb..f1654e55b7e 100644 --- a/spec/support/helpers/stub_feature_flags.rb +++ b/spec/support/helpers/stub_feature_flags.rb @@ -70,4 +70,18 @@ module StubFeatureFlags def skip_default_enabled_yaml_check allow(Feature::Definition).to receive(:default_enabled?).and_return(false) end + + def stub_feature_flag_definition(name, opts = {}) + opts = opts.with_defaults( + name: name, + type: 'development', + default_enabled: false + ) + + Feature::Definition.new("#{opts[:type]}/#{name}.yml", opts).tap do |definition| + all_definitions = Feature::Definition.definitions + all_definitions[definition.key] = definition + allow(Feature::Definition).to receive(:definitions).and_return(all_definitions) + end + end end