diff --git a/app/assets/stylesheets/framework/forms.scss b/app/assets/stylesheets/framework/forms.scss index 4d0d64ae723..50783433c3d 100644 --- a/app/assets/stylesheets/framework/forms.scss +++ b/app/assets/stylesheets/framework/forms.scss @@ -83,7 +83,6 @@ label { margin-right: 0; .form-control { - height: 29px; background: $white; font-family: $monospace-font; } diff --git a/app/controllers/autocomplete_controller.rb b/app/controllers/autocomplete_controller.rb index 663e3cf8648..eb6cedb5d4a 100644 --- a/app/controllers/autocomplete_controller.rb +++ b/app/controllers/autocomplete_controller.rb @@ -14,6 +14,7 @@ class AutocompleteController < ApplicationController urgency :low, [:merge_request_target_branches] urgency :default, [:users] + urgency :medium, [:projects] def users group = Autocomplete::GroupFinder diff --git a/app/controllers/explore/projects_controller.rb b/app/controllers/explore/projects_controller.rb index 335ccce5507..796ef5f0a89 100644 --- a/app/controllers/explore/projects_controller.rb +++ b/app/controllers/explore/projects_controller.rb @@ -26,8 +26,7 @@ class Explore::ProjectsController < Explore::ApplicationController feature_category :projects # TODO: Set higher urgency after addressing https://gitlab.com/gitlab-org/gitlab/-/issues/357913 # and https://gitlab.com/gitlab-org/gitlab/-/issues/358945 - urgency :low, [:index, :trending] - urgency :low, [:index] + urgency :low, [:index, :topics, :trending] def index show_alert_if_search_is_disabled diff --git a/app/controllers/groups_controller.rb b/app/controllers/groups_controller.rb index 700042614cc..199e14fa949 100644 --- a/app/controllers/groups_controller.rb +++ b/app/controllers/groups_controller.rb @@ -61,7 +61,7 @@ class GroupsController < Groups::ApplicationController urgency :high, [:unfoldered_environment_names] # TODO: Set #show to higher urgency after resolving https://gitlab.com/gitlab-org/gitlab/-/issues/334795 - urgency :low, [:merge_requests, :show, :create, :new, :update] + urgency :low, [:merge_requests, :show, :create, :new, :update, :projects, :destroy] def index redirect_to(current_user ? dashboard_groups_path : explore_groups_path) diff --git a/app/controllers/projects/learn_gitlab_controller.rb b/app/controllers/projects/learn_gitlab_controller.rb index b9f9a1810b7..61e4a1812ba 100644 --- a/app/controllers/projects/learn_gitlab_controller.rb +++ b/app/controllers/projects/learn_gitlab_controller.rb @@ -7,6 +7,7 @@ class Projects::LearnGitlabController < Projects::ApplicationController before_action :enable_video_tutorials_continuous_onboarding_experiment feature_category :users + urgency :low, [:index] def index end diff --git a/app/controllers/projects_controller.rb b/app/controllers/projects_controller.rb index 7986b42b2f8..621bc64326a 100644 --- a/app/controllers/projects_controller.rb +++ b/app/controllers/projects_controller.rb @@ -57,8 +57,7 @@ class ProjectsController < Projects::ApplicationController feature_category :portfolio_management, [:planning_hierarchy] # TODO: Set high urgency for #show https://gitlab.com/gitlab-org/gitlab/-/issues/334444 - urgency :low, [:refs, :show, :destroy] - + urgency :low, [:refs, :show, :toggle_star, :transfer, :archive, :destroy] urgency :high, [:unfoldered_environment_names] def index diff --git a/app/views/projects/services/mattermost_slash_commands/_detailed_help.html.haml b/app/views/projects/services/mattermost_slash_commands/_detailed_help.html.haml index 9d8ce186232..fec443738c3 100644 --- a/app/views/projects/services/mattermost_slash_commands/_detailed_help.html.haml +++ b/app/views/projects/services/mattermost_slash_commands/_detailed_help.html.haml @@ -34,14 +34,14 @@ .col-12.input-group = text_field_tag :display_name, "GitLab / #{pretty_name}".html_safe, class: 'form-control form-control-sm', readonly: 'readonly' .input-group-append - = clipboard_button(target: '#display_name', class: 'input-group-text') + = clipboard_button(target: '#display_name', class: 'gl-button btn-default btn-icon input-group-text') .form-group = label_tag :description, _('Description'), class: 'col-12 col-form-label label-bold' .col-12.input-group = text_field_tag :description, run_actions_text, class: 'form-control form-control-sm', readonly: 'readonly' .input-group-append - = clipboard_button(target: '#description', class: 'input-group-text') + = clipboard_button(target: '#description', class: 'gl-button btn-default btn-icon input-group-text') .form-group = label_tag nil, s_('MattermostService|Command trigger word'), class: 'col-12 col-form-label label-bold' @@ -59,7 +59,7 @@ .col-12.input-group = text_field_tag :request_url, service_trigger_url(integration), class: 'form-control form-control-sm', readonly: 'readonly' .input-group-append - = clipboard_button(target: '#request_url', class: 'input-group-text') + = clipboard_button(target: '#request_url', class: 'gl-button btn-default btn-icon input-group-text') .form-group = label_tag nil, s_('MattermostService|Request method'), class: 'col-12 col-form-label label-bold' @@ -70,14 +70,14 @@ .col-12.input-group = text_field_tag :response_username, 'GitLab', class: 'form-control form-control-sm', readonly: 'readonly' .input-group-append - = clipboard_button(target: '#response_username', class: 'input-group-text') + = clipboard_button(target: '#response_username', class: 'gl-button btn-default btn-icon input-group-text') .form-group = label_tag :response_icon, s_('MattermostService|Response icon'), class: 'col-12 col-form-label label-bold' .col-12.input-group = text_field_tag :response_icon, asset_url('gitlab_logo.png'), class: 'form-control form-control-sm', readonly: 'readonly' .input-group-append - = clipboard_button(target: '#response_icon', class: 'input-group-text') + = clipboard_button(target: '#response_icon', class: 'gl-button btn-default btn-icon input-group-text') .form-group = label_tag nil, _('Autocomplete'), class: 'col-12 col-form-label label-bold' @@ -88,11 +88,11 @@ .col-12.input-group = text_field_tag :autocomplete_hint, '[help]', class: 'form-control form-control-sm', readonly: 'readonly' .input-group-append - = clipboard_button(target: '#autocomplete_hint', class: 'input-group-text') + = clipboard_button(target: '#autocomplete_hint', class: 'gl-button btn-default btn-icon input-group-text') .form-group = label_tag :autocomplete_description, _('Autocomplete description'), class: 'col-12 col-form-label label-bold' .col-12.input-group = text_field_tag :autocomplete_description, run_actions_text, class: 'form-control form-control-sm', readonly: 'readonly' .input-group-append - = clipboard_button(target: '#autocomplete_description', class: 'input-group-text') + = clipboard_button(target: '#autocomplete_description', class: 'gl-button btn-default btn-icon input-group-text') diff --git a/db/docs/member_tasks.yml b/db/docs/member_tasks.yml index 7f55c8d6eec..8802d50176b 100644 --- a/db/docs/member_tasks.yml +++ b/db/docs/member_tasks.yml @@ -3,7 +3,7 @@ table_name: member_tasks classes: - MemberTask feature_categories: -- authentication_and_authorization +- experimentation_activation description: TODO introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/69299 milestone: '14.5' diff --git a/db/migrate/20220413124200_add_view_for_per_table_autovacuum_status.rb b/db/migrate/20220413124200_add_view_for_per_table_autovacuum_status.rb new file mode 100644 index 00000000000..2b49d6d18b2 --- /dev/null +++ b/db/migrate/20220413124200_add_view_for_per_table_autovacuum_status.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +class AddViewForPerTableAutovacuumStatus < Gitlab::Database::Migration[1.0] + def up + execute <<~SQL + CREATE OR REPLACE FUNCTION postgres_pg_stat_activity_autovacuum() RETURNS SETOF pg_catalog.pg_stat_activity AS + $$ + SELECT * + FROM pg_stat_activity + WHERE datname = current_database() + AND state = 'active' + AND backend_type = 'autovacuum worker' + $$ + LANGUAGE sql + VOLATILE + SECURITY DEFINER + SET search_path = 'pg_catalog', 'pg_temp'; + + CREATE VIEW postgres_autovacuum_activity AS + WITH processes as + ( + SELECT query, query_start, (regexp_matches(query, '^autovacuum: VACUUM (\w+)\.(\w+)')) as matches + FROM postgres_pg_stat_activity_autovacuum() + WHERE query ~* '^autovacuum: VACUUM \w+\.\w+' + ) + SELECT matches[1] || '.' || matches[2] as table_identifier, + matches[1] as schema, + matches[2] as table, + query_start as vacuum_start + FROM processes; + + COMMENT ON VIEW postgres_autovacuum_activity IS 'Contains information about PostgreSQL backends currently performing autovacuum operations on the tables indicated here.'; + SQL + end + + def down + execute <<~SQL + DROP VIEW IF EXISTS postgres_autovacuum_activity; + DROP FUNCTION IF EXISTS postgres_pg_stat_activity_autovacuum; + SQL + end +end diff --git a/db/schema_migrations/20220413124200 b/db/schema_migrations/20220413124200 new file mode 100644 index 00000000000..cbf9676b231 --- /dev/null +++ b/db/schema_migrations/20220413124200 @@ -0,0 +1 @@ +f5d7722ee4b0288294b08b470e7f9c072cf50f888a33fa7ed234eaafa5684843 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 971178bb754..00fd254b7a9 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -106,6 +106,17 @@ BEGIN END; $$; +CREATE FUNCTION postgres_pg_stat_activity_autovacuum() RETURNS SETOF pg_stat_activity + LANGUAGE sql SECURITY DEFINER + SET search_path TO 'pg_catalog', 'pg_temp' + AS $$ + SELECT * + FROM pg_stat_activity + WHERE datname = current_database() + AND state = 'active' + AND backend_type = 'autovacuum worker' +$$; + CREATE FUNCTION set_has_external_issue_tracker() RETURNS trigger LANGUAGE plpgsql AS $$ @@ -18728,6 +18739,22 @@ CREATE SEQUENCE postgres_async_indexes_id_seq ALTER SEQUENCE postgres_async_indexes_id_seq OWNED BY postgres_async_indexes.id; +CREATE VIEW postgres_autovacuum_activity AS + WITH processes AS ( + SELECT postgres_pg_stat_activity_autovacuum.query, + postgres_pg_stat_activity_autovacuum.query_start, + regexp_matches(postgres_pg_stat_activity_autovacuum.query, '^autovacuum: VACUUM (w+).(w+)'::text) AS matches + FROM postgres_pg_stat_activity_autovacuum() postgres_pg_stat_activity_autovacuum(datid, datname, pid, usesysid, usename, application_name, client_addr, client_hostname, client_port, backend_start, xact_start, query_start, state_change, wait_event_type, wait_event, state, backend_xid, backend_xmin, query, backend_type) + WHERE (postgres_pg_stat_activity_autovacuum.query ~* '^autovacuum: VACUUM w+.w+'::text) + ) + SELECT ((processes.matches[1] || '.'::text) || processes.matches[2]) AS table_identifier, + processes.matches[1] AS schema, + processes.matches[2] AS "table", + processes.query_start AS vacuum_start + FROM processes; + +COMMENT ON VIEW postgres_autovacuum_activity IS 'Contains information about PostgreSQL backends currently performing autovacuum operations on the tables indicated here.'; + CREATE VIEW postgres_foreign_keys AS SELECT pg_constraint.oid, pg_constraint.conname AS name, diff --git a/doc/administration/nfs.md b/doc/administration/nfs.md index ef5d26ac845..340d8b5c93e 100644 --- a/doc/administration/nfs.md +++ b/doc/administration/nfs.md @@ -24,9 +24,9 @@ file system performance, see Starting with GitLab version 14.0, support for NFS to store Git repository data is deprecated. Technical customer support and engineering support is available for the 14.x releases. Engineering is fixing bugs and security vulnerabilities consistent with our [release and maintenance policy](../policy/maintenance.md#security-releases). -Upon the release of GitLab 15.0 (tentatively May 22nd, 2022) technical and engineering support for using NFS to store Git repository data will be officially at end-of-life. There will be no product changes or troubleshooting provided via Engineering, Security or Paid Support channels after the release date of 15.0, regardless of your GitLab version. +Upon the release of GitLab 15.6 technical and engineering support for using NFS to store Git repository data will be officially at end-of-life. There will be no product changes or troubleshooting provided via Engineering, Security or Paid Support channels after the release date of 15.6, regardless of your GitLab version. -Until the release of 15.0, for customers running 14.x releases, we continue to help with Git related tickets from customers running one or more Gitaly servers with its data stored on NFS. Examples may include: +Until the release of 15.6, for customers running 14.x releases, we continue to help with Git related tickets from customers running one or more Gitaly servers with its data stored on NFS. Examples may include: - Performance issues or timeouts accessing Git data - Commits or branches vanish @@ -39,10 +39,10 @@ Assistance is limited to activities like: - Verifying that NFS client mount options match our [documented recommendations](#mount-options) - Analyzing the GitLab Workhorse and Rails logs, and determining that `500` errors being seen in the environment are caused by slow responses from Gitaly -GitLab support is unable to continue with the investigation if: +GitLab support is unable to continue with the investigation if both: -- The date of the request is on or after the release of GitLab version 15.0, and -- Support Engineers and Management determine that all reasonable non-NFS root causes have been exhausted +- The date of the request is on or after the release of GitLab version 15.6. +- Support Engineers and Management determine that all reasonable non-NFS root causes have been exhausted. If the issue is reproducible, or if it happens intermittently but regularly, GitLab Support can investigate providing the issue reproduces without the use of NFS. In order to reproduce without NFS, the affected repositories should be migrated to a different Gitaly shard, such as Gitaly cluster or a standalone Gitaly VM, backed with block storage. diff --git a/doc/ci/troubleshooting.md b/doc/ci/troubleshooting.md index 81cb924532c..dafa9188aa2 100644 --- a/doc/ci/troubleshooting.md +++ b/doc/ci/troubleshooting.md @@ -331,6 +331,12 @@ busy_resources.pluck(:build_id) busy_resources.update_all(build_id: nil) ``` +### Job log slow to update + +When you visit the job log page for a running job, there could be a delay of up to +60 seconds before the log updates. The default refresh time is 60 seconds, but after +the log is viewed in the UI, the following log updates should occur every 3 seconds. + ## How to get help If you are unable to resolve pipeline issues, you can get help from: diff --git a/doc/development/secure_coding_guidelines.md b/doc/development/secure_coding_guidelines.md index 8a86a46d1d3..2b70ee395a2 100644 --- a/doc/development/secure_coding_guidelines.md +++ b/doc/development/secure_coding_guidelines.md @@ -1210,3 +1210,36 @@ An example of well implemented `Gitlab::UrlBlocker.validate!` call that prevents ### Resources - [CWE-367: Time-of-check Time-of-use (TOCTOU) Race Condition](https://cwe.mitre.org/data/definitions/367.html) + +## Handling credentials + +Credentials can be: + +- Login details like username and password. +- Private keys. +- Tokens (PAT, runner tokens, JWT token, CSRF tokens, project access tokens, etc). +- Session cookies. +- Any other piece of information that can be used for authentication or authorization purposes. + +This sensitive data must be handled carefully to avoid leaks which could lead to unauthorized access. If you have questions or need help with any of the following guidance, talk to the GitLab AppSec team on Slack (`#sec-appsec`). + +### At rest + +- Credentials must be encrypted while at rest (database or file) with `attr_encrypted`. See [issue #26243](https://gitlab.com/gitlab-org/gitlab/-/issues/26243) before using `attr_encrypted`. + - Store the encryption keys separately from the encrypted credentials with proper access control. For instance, store the keys in a vault, KMS, or file. Here is an [example](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/models/user.rb#L70-74) use of `attr_encrypted` for encryption with keys stored in separate access controlled file. + - When the intention is to only compare secrets, store only the salted hash of the secret instead of the encrypted value. +- Never commit credentials to repositories. + - The [Gitleaks Git hook](https://gitlab.com/gitlab-com/gl-security/security-research/gitleaks-endpoint-installer) is recommended for preventing credentials from being committed. +- Never log credentials under any circumstance. Issue [#353857](https://gitlab.com/gitlab-org/gitlab/-/issues/353857) is an example of credential leaks through log file. +- When credentials are required in a CI/CD job, use [masked variables](../ci/variables/index.md#mask-a-cicd-variable) to help prevent accidental exposure in the job logs. Be aware that when [debug logging](../ci/variables/index.md#debug-logging) is enabled, all masked CI/CD variables are visible in job logs. Also consider using [protected variables](../ci/variables/index.md#protect-a-cicd-variable) when possible so that sensitive CI/CD variables are only available to pipelines running on protected branches or tags. +- Proper scanners must be enabled depending on what data those credentials are protecting. See the [Application Security Inventory Policy](https://about.gitlab.com/handbook/engineering/security/security-engineering-and-research/application-security/inventory.html#policies) and our [Data Classification Standards](https://about.gitlab.com/handbook/engineering/security/data-classification-standard.html#data-classification-standards). +- To store and/or share credentials between teams, refer to [1Password for Teams](https://about.gitlab.com/handbook/security/#1password-for-teams) and follow [the 1Password Guidelines](https://about.gitlab.com/handbook/security/#1password-guidelines). +- If you need to share a secret with a team member, use 1Password. Do not share a secret over email, Slack, or other service on the Internet. + +### In transit + +- Use an encrypted channel like TLS to transmit credentials. See [our TLS minimum recommendation guidelines](#tls-minimum-recommended-version). +- Avoid including credentials as part of an HTTP response unless it is absolutely necessary as part of the workflow. For example, generating a PAT for users. +- Avoid sending credentials in URL parameters, as these can be more easily logged inadvertently during transit. + +In the event of credential leak through an MR, issue, or any other medium, [reach out to SIRT team](https://about.gitlab.com/handbook/engineering/security/security-operations/sirt/#-engaging-sirt). diff --git a/lib/api/badges.rb b/lib/api/badges.rb index ca960a179ab..aaf5be1d9f7 100644 --- a/lib/api/badges.rb +++ b/lib/api/badges.rb @@ -72,7 +72,10 @@ module API params do requires :badge_id, type: Integer, desc: 'The badge ID' end - get ":id/badges/:badge_id", urgency: :default do + # TODO: Set PUT /projects/:id/badges/:badge_id to low urgency and GET to default urgency + # after different urgencies are supported for different HTTP verbs. + # See https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1670 + get ":id/badges/:badge_id", urgency: :low do source = find_source(source_type, params[:id]) badge = find_badge(source) diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 0ed14476c61..a56f1ad44ad 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -266,7 +266,7 @@ module API end desc 'Remove a group.' - delete ":id", feature_category: :subgroups do + delete ":id", feature_category: :subgroups, urgency: :low do group = find_group!(params[:id]) authorize! :admin_group, group check_subscription! group diff --git a/lib/api/projects.rb b/lib/api/projects.rb index ab90b60224e..398e392277e 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -231,7 +231,7 @@ module API use :collection_params use :statistics_params end - get ":user_id/starred_projects", feature_category: :projects do + get ":user_id/starred_projects", feature_category: :projects, urgency: :low do user = find_user(params[:user_id]) not_found!('User') unless user @@ -469,7 +469,7 @@ module API desc 'Unarchive a project' do success Entities::Project end - post ':id/unarchive', feature_category: :projects do + post ':id/unarchive', feature_category: :projects, urgency: :default do authorize!(:archive_project, user_project) ::Projects::UpdateService.new(user_project, current_user, archived: false).execute diff --git a/lib/api/users.rb b/lib/api/users.rb index 9c62ac69eb4..096b8da7461 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -1122,7 +1122,7 @@ module API optional :show_whitespace_in_diffs, type: Boolean, desc: 'Flag indicating the user sees whitespace changes in diffs' at_least_one_of :view_diffs_file_by_file, :show_whitespace_in_diffs end - put "preferences", feature_category: :users do + put "preferences", feature_category: :users, urgency: :high do authenticate! preferences = current_user.user_preference diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index ae0ea919b62..b1869574a2c 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -388,6 +388,7 @@ plan_limits: :gitlab_main plans: :gitlab_main pool_repositories: :gitlab_main postgres_async_indexes: :gitlab_shared +postgres_autovacuum_activity: :gitlab_shared postgres_foreign_keys: :gitlab_shared postgres_index_bloat_estimates: :gitlab_shared postgres_indexes: :gitlab_shared