From 0c2ef1f35c7ee0abc9e9d1d322c6c654a6d8a4d0 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Mon, 7 Jun 2021 03:09:56 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- .gitignore | 1 + app/controllers/admin/dashboard_controller.rb | 2 +- app/controllers/health_controller.rb | 1 + config/README.md | 31 +++++++++++ config/initializers/7_redis.rb | 1 + config/redis.trace_chunks.yml.example | 38 +++++++++++++ .../operations/extra_sidekiq_processes.md | 36 ++++++------ .../application_security/api_fuzzing/index.md | 42 ++++++++++++++ .../application_security/dast_api/index.md | 50 +++++++++++++++-- lib/gitlab/health_checks/redis/redis_check.rb | 3 +- .../health_checks/redis/trace_chunks_check.rb | 35 ++++++++++++ lib/gitlab/instrumentation/redis.rb | 3 +- lib/gitlab/redis/trace_chunks.rb | 12 ++++ lib/gitlab/redis/wrapper.rb | 15 +++++ scripts/prepare_build.sh | 3 + .../redis/trace_chunks_check_spec.rb | 8 +++ spec/lib/gitlab/instrumentation/redis_spec.rb | 3 +- spec/lib/gitlab/redis/trace_chunks_spec.rb | 55 +++++++++++++++++++ 18 files changed, 313 insertions(+), 26 deletions(-) create mode 100644 config/redis.trace_chunks.yml.example create mode 100644 lib/gitlab/health_checks/redis/trace_chunks_check.rb create mode 100644 lib/gitlab/redis/trace_chunks.rb create mode 100644 spec/lib/gitlab/health_checks/redis/trace_chunks_check_spec.rb create mode 100644 spec/lib/gitlab/redis/trace_chunks_spec.rb diff --git a/.gitignore b/.gitignore index 9d9730bf406..323741575de 100644 --- a/.gitignore +++ b/.gitignore @@ -42,6 +42,7 @@ eslint-report.html /config/redis.cache.yml /config/redis.queues.yml /config/redis.shared_state.yml +/config/redis.trace_chunks.yml /config/unicorn.rb /config/puma.rb /config/puma_actioncable.rb diff --git a/app/controllers/admin/dashboard_controller.rb b/app/controllers/admin/dashboard_controller.rb index 46e5a508a1b..ba24e3e619b 100644 --- a/app/controllers/admin/dashboard_controller.rb +++ b/app/controllers/admin/dashboard_controller.rb @@ -15,7 +15,7 @@ class Admin::DashboardController < Admin::ApplicationController @groups = Group.order_id_desc.with_route.limit(10) @notices = Gitlab::ConfigChecker::PumaRuggedChecker.check @notices += Gitlab::ConfigChecker::ExternalDatabaseChecker.check - @redis_versions = [Gitlab::Redis::Queues, Gitlab::Redis::SharedState, Gitlab::Redis::Cache].map(&:version).uniq + @redis_versions = [Gitlab::Redis::Queues, Gitlab::Redis::SharedState, Gitlab::Redis::Cache, Gitlab::Redis::TraceChunks].map(&:version).uniq end # rubocop: enable CodeReuse/ActiveRecord diff --git a/app/controllers/health_controller.rb b/app/controllers/health_controller.rb index c6a02250896..99b0b775217 100644 --- a/app/controllers/health_controller.rb +++ b/app/controllers/health_controller.rb @@ -15,6 +15,7 @@ class HealthController < ActionController::Base Gitlab::HealthChecks::Redis::CacheCheck, Gitlab::HealthChecks::Redis::QueuesCheck, Gitlab::HealthChecks::Redis::SharedStateCheck, + Gitlab::HealthChecks::Redis::TraceChunksCheck, Gitlab::HealthChecks::GitalyCheck ].freeze diff --git a/config/README.md b/config/README.md index 7f3125cefd2..be5bd442fd8 100644 --- a/config/README.md +++ b/config/README.md @@ -147,3 +147,34 @@ searched): 3. the configuration file pointed to by the `GITLAB_REDIS_CONFIG_FILE` environment variable 4. the configuration file `resque.yml` + +## redis.trace_chunks.yml + +If configured, `redis.trace_chunks.yml` overrides the +`resque.yml` settings to configure the Redis database instance +used for clients of `::Gitlab::Redis::TraceChunks` which stores CI trace chunks. + +Settings here can be overridden by the environment variable +`GITLAB_REDIS_TRACE_CHUNKS_CONFIG_FILE` which provides +an alternate location for configuration settings. + +The order of precedence for the URL used to connect to the Redis instance +used for `trace_chunks` is: +1. URL from a configuration file pointed to by the +`GITLAB_REDIS_TRACE_CHUNKS_CONFIG_FILE` environment variable +2. URL from `redis.trace_chunks.yml` +3. URL from a configuration file pointed to by the +`GITLAB_REDIS_CONFIG_FILE` environment variable +4. URL from `resque.yml` +5. `redis://localhost:6383` + +The order of precedence for all other configuration settings for `trace_chunks` +are selected from only the first of the following files found (if a setting +is not provided in an earlier file, the remainder of the files are not +searched): +1. the configuration file pointed to by the +`GITLAB_REDIS_TRACE_CHUNKS_CONFIG_FILE` environment variable +2. the configuration file `redis.trace_chunks.yml` +3. the configuration file pointed to by the +`GITLAB_REDIS_CONFIG_FILE` environment variable +4. the configuration file `resque.yml` diff --git a/config/initializers/7_redis.rb b/config/initializers/7_redis.rb index a6025a6dbf0..fe37dfd7579 100644 --- a/config/initializers/7_redis.rb +++ b/config/initializers/7_redis.rb @@ -8,3 +8,4 @@ Gitlab::Redis::Cache.with { nil } Gitlab::Redis::Queues.with { nil } Gitlab::Redis::SharedState.with { nil } +Gitlab::Redis::TraceChunks.with { nil } diff --git a/config/redis.trace_chunks.yml.example b/config/redis.trace_chunks.yml.example new file mode 100644 index 00000000000..d38b9ba4966 --- /dev/null +++ b/config/redis.trace_chunks.yml.example @@ -0,0 +1,38 @@ +# If you change this file in a merge request, please also create +# a merge request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests +# +development: + url: redis://localhost:6379/13 + # + # url: redis://localhost:6382 + # sentinels: + # - + # host: localhost + # port: 26382 # point to sentinel, not to redis port + # - + # host: replica2 + # port: 26382 # point to sentinel, not to redis port +test: + url: redis://localhost:6379/13 + # + # url: redis://localhost:6382 +production: + # Redis (single instance) + url: unix:/var/run/redis/redis.trace_chunks.sock + ## + # Redis + Sentinel (for HA) + # + # Please read instructions carefully before using it as you may lose data: + # http://redis.io/topics/sentinel + # + # You must specify a list of a few sentinels that will handle client connection + # please read here for more information: https://docs.gitlab.com/ee/administration/redis/index.html + ## + # url: redis://master:6382 + # sentinels: + # - + # host: replica1 + # port: 26382 # point to sentinel, not to redis port + # - + # host: replica2 + # port: 26382 # point to sentinel, not to redis port diff --git a/doc/administration/operations/extra_sidekiq_processes.md b/doc/administration/operations/extra_sidekiq_processes.md index cf058a85e72..a4235afad04 100644 --- a/doc/administration/operations/extra_sidekiq_processes.md +++ b/doc/administration/operations/extra_sidekiq_processes.md @@ -70,11 +70,11 @@ To start multiple processes: ] ``` - `*` cannot be combined with concrete queue names - `*, mailers` will - just handle the `mailers` queue. + `*` cannot be combined with concrete queue names - `*, mailers` + just handles the `mailers` queue. When `sidekiq-cluster` is only running on a single node, make sure that at least - one process is running on all queues using `*`. This means a process will + one process is running on all queues using `*`. This means a process is This includes queues that have dedicated processes. If `sidekiq-cluster` is running on more than one node, you can also use @@ -142,7 +142,7 @@ WARNING: Sidekiq cluster is [scheduled](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/240) to be the only way to start Sidekiq in GitLab 14.0. -By default, the Sidekiq service will run `sidekiq-cluster`. To disable this behavior, +By default, the Sidekiq service runs `sidekiq-cluster`. To disable this behavior, add the following to the Sidekiq configuration: ```ruby @@ -151,7 +151,7 @@ sidekiq['cluster'] = false ``` All of the aforementioned configuration options for `sidekiq` -are available. By default, they will be configured as follows: +are available. By default, they are configured as follows: ```ruby sidekiq['queue_selector'] = false @@ -168,14 +168,14 @@ cluster as above. When disabling `sidekiq_cluster`, you must copy your configuration for `sidekiq_cluster`over to `sidekiq`. Anything configured for -`sidekiq_cluster` will be overridden by the options for `sidekiq` when +`sidekiq_cluster` is overridden by the options for `sidekiq` when setting `sidekiq['cluster'] = true`. -When using this feature, the service called `sidekiq` will now be +When using this feature, the service called `sidekiq` is now running `sidekiq-cluster`. The [concurrency](#manage-concurrency) and other options configured -for Sidekiq will be respected. +for Sidekiq are respected. By default, logs for `sidekiq-cluster` go to `/var/log/gitlab/sidekiq` like regular Sidekiq logs. @@ -220,7 +220,7 @@ use all of its resources to perform those operations. To set up a separate Each process defined under `sidekiq` starts with a number of threads that equals the number of queues, plus one spare thread. For example, a process that handles the `process_commit` and `post_receive` -queues will use three threads in total. +queues uses three threads in total. ## Manage concurrency @@ -251,16 +251,16 @@ Running Sidekiq cluster is the default in GitLab 13.0 and later. ``` `min_concurrency` and `max_concurrency` are independent; one can be set without -the other. Setting `min_concurrency` to `0` will disable the limit. +the other. Setting `min_concurrency` to `0` disables the limit. For each queue group, let `N` be one more than the number of queues. The -concurrency factor will be set to: +concurrency factor are set to: 1. `N`, if it's between `min_concurrency` and `max_concurrency`. 1. `max_concurrency`, if `N` exceeds this value. 1. `min_concurrency`, if `N` is less than this value. -If `min_concurrency` is equal to `max_concurrency`, then this value will be used +If `min_concurrency` is equal to `max_concurrency`, then this value is used regardless of the number of queues. When `min_concurrency` is greater than `max_concurrency`, it is treated as @@ -287,7 +287,7 @@ Running Sidekiq directly is scheduled to be removed in GitLab sudo gitlab-ctl reconfigure ``` -This will set the concurrency (number of threads) for the Sidekiq process. +This sets the concurrency (number of threads) for the Sidekiq process. ## Modify the check interval @@ -353,21 +353,21 @@ you'd use the following: ### Monitor the `sidekiq-cluster` command -The `sidekiq-cluster` command will not terminate once it has started the desired -amount of Sidekiq processes. Instead, the process will continue running and +The `sidekiq-cluster` command does not terminate once it has started the desired +amount of Sidekiq processes. Instead, the process continues running and forward any signals to the child processes. This makes it easy to stop all Sidekiq processes as you simply send a signal to the `sidekiq-cluster` process, instead of having to send it to the individual processes. If the `sidekiq-cluster` process crashes or receives a `SIGKILL`, the child -processes will terminate themselves after a few seconds. This ensures you don't +processes terminate themselves after a few seconds. This ensures you don't end up with zombie Sidekiq processes. All of this makes monitoring the processes fairly easy. Simply hook up `sidekiq-cluster` to your supervisor of choice (for example, runit) and you're good to go. -If a child process died the `sidekiq-cluster` command will signal all remaining +If a child process died the `sidekiq-cluster` command signals all remaining process to terminate, then terminate itself. This removes the need for `sidekiq-cluster` to re-implement complex process monitoring/restarting code. Instead you should make sure your supervisor restarts the `sidekiq-cluster` @@ -383,7 +383,7 @@ file is written, but this can be changed by passing the `--pidfile` option to /opt/gitlab/embedded/service/gitlab-rails/bin/sidekiq-cluster --pidfile /var/run/gitlab/sidekiq_cluster.pid process_commit ``` -Keep in mind that the PID file will contain the PID of the `sidekiq-cluster` +Keep in mind that the PID file contains the PID of the `sidekiq-cluster` command and not the PID(s) of the started Sidekiq processes. ### Environment diff --git a/doc/user/application_security/api_fuzzing/index.md b/doc/user/application_security/api_fuzzing/index.md index abfa65e636d..22487d14e86 100644 --- a/doc/user/application_security/api_fuzzing/index.md +++ b/doc/user/application_security/api_fuzzing/index.md @@ -568,6 +568,7 @@ profile increases as the number of tests increases. | `FUZZAPI_TARGET_URL` | Base URL of API testing target. | | `FUZZAPI_CONFIG` | [Deprecated](https://gitlab.com/gitlab-org/gitlab/-/issues/276395) in GitLab 13.12, replaced with default `.gitlab/gitlab-api-fuzzing-config.yml`. API Fuzzing configuration file. | |[`FUZZAPI_PROFILE`](#api-fuzzing-profiles) | Configuration profile to use during testing. Defaults to `Quick-10`. | +|[`FUZZAPI_EXCLUDE_PATHS`](#exclude-paths) | Exclude API URL paths from testing. | |[`FUZZAPI_OPENAPI`](#openapi-specification) | OpenAPI Specification file or URL. | |[`FUZZAPI_HAR`](#http-archive-har) | HTTP Archive (HAR) file. | |[`FUZZAPI_POSTMAN_COLLECTION`](#postman-collection) | Postman Collection file. | @@ -825,6 +826,47 @@ variables: FUZZAPI_OVERRIDES_INTERVAL: 300 ``` +### Exclude Paths + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/211892) in GitLab 14.0. + +When testing an API it can be useful to exclude certain paths. For example, you might exclude testing of an authentication service or an older version of the API. To exclude paths, use the `FUZZAPI_EXCLUDE_PATHS` CI/CD variable . This variable is specified in your `.gitlab-ci.yml` file. To exclude multiple paths, separate entries using the `;` character. In the provided paths you can use a single character wildcard `?` and `*` for a multiple character wildcard. + +To verify the paths are excluded, review the `Tested Operations` and `Excluded Operations` portion of the job output. You should not see any excluded paths listed under `Tested Operations`. + +```plaintext +2021-05-27 21:51:08 [INF] API Security: --[ Tested Operations ]------------------------- +2021-05-27 21:51:08 [INF] API Security: 201 POST http://target:7777/api/users CREATED +2021-05-27 21:51:08 [INF] API Security: ------------------------------------------------ +2021-05-27 21:51:08 [INF] API Security: --[ Excluded Operations ]----------------------- +2021-05-27 21:51:08 [INF] API Security: GET http://target:7777/api/messages +2021-05-27 21:51:08 [INF] API Security: POST http://target:7777/api/messages +2021-05-27 21:51:08 [INF] API Security: ------------------------------------------------ +``` + +#### Examples of excluding paths + +This example excludes the `/auth` resource. This does not exclude child resources (`/auth/child`). + +```yaml +variables: + FUZZAPI_EXCLUDE_PATHS=/auth +``` + +To exclude `/auth`, and child resources (`/auth/child`), we use a wildcard. + +```yaml +variables: + FUZZAPI_EXCLUDE_PATHS=/auth* +``` + +To exclude multiple paths we can use the `;` character. In this example we exclude `/auth*` and `/v1/*`. + +```yaml +variables: + FUZZAPI_EXCLUDE_PATHS=/auth*;/v1/* +``` + ### Header Fuzzing Header fuzzing is disabled by default due to the high number of false positives that occur with many diff --git a/doc/user/application_security/dast_api/index.md b/doc/user/application_security/dast_api/index.md index e8e5d7a3957..b967b80d85a 100644 --- a/doc/user/application_security/dast_api/index.md +++ b/doc/user/application_security/dast_api/index.md @@ -72,8 +72,8 @@ starting in GitLab 14.0, GitLab will not check your repository's root for config > Support for OpenAPI Specification using YAML format was > [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/330583) in GitLab 14.0. -The [OpenAPI Specification](https://www.openapis.org/) (formerly the Swagger Specification) is an API description format for REST APIs. -This section shows you how to configure API fuzzing using an OpenAPI Specification to provide information about the target API to test. +The [OpenAPI Specification](https://www.openapis.org/) (formerly the Swagger Specification) is an API description format for REST APIs. +This section shows you how to configure API fuzzing using an OpenAPI Specification to provide information about the target API to test. OpenAPI Specifications are provided as a file system resource or URL. Both JSON and YAML OpenAPI formats are supported. DAST API uses an OpenAPI document to generate the request body. When a request body is required, @@ -635,6 +635,7 @@ can be added, removed, and modified by creating a custom configuration. | `DAST_API_TARGET_URL` | Base URL of API testing target. | |[`DAST_API_CONFIG`](#configuration-files) | DAST API configuration file. Defaults to `.gitlab-dast-api.yml`. | |[`DAST_API_PROFILE`](#configuration-files) | Configuration profile to use during testing. Defaults to `Quick`. | +|[`FUZZAPI_EXCLUDE_PATHS`](#exclude-paths) | Exclude API URL paths from testing. | |[`DAST_API_OPENAPI`](#openapi-specification) | OpenAPI specification file or URL. | |[`DAST_API_HAR`](#http-archive-har) | HTTP Archive (HAR) file. | |[`DAST_API_POSTMAN_COLLECTION`](#postman-collection) | Postman Collection file. | @@ -894,6 +895,47 @@ variables: DAST_API_OVERRIDES_INTERVAL: 300 ``` +### Exclude Paths + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/211892) in GitLab 14.0. + +When testing an API it can be useful to exclude certain paths. For example, you might exclude testing of an authentication service or an older version of the API. To exclude paths, use the `FUZZAPI_EXCLUDE_PATHS` CI/CD variable . This variable is specified in your `.gitlab-ci.yml` file. To exclude multiple paths, separate entries using the `;` character. In the provided paths you can use a single character wildcard `?` and `*` for a multiple character wildcard. + +To verify the paths are excluded, review the `Tested Operations` and `Excluded Operations` portion of the job output. You should not see any excluded paths listed under `Tested Operations`. + +```plaintext +2021-05-27 21:51:08 [INF] API Security: --[ Tested Operations ]------------------------- +2021-05-27 21:51:08 [INF] API Security: 201 POST http://target:7777/api/users CREATED +2021-05-27 21:51:08 [INF] API Security: ------------------------------------------------ +2021-05-27 21:51:08 [INF] API Security: --[ Excluded Operations ]----------------------- +2021-05-27 21:51:08 [INF] API Security: GET http://target:7777/api/messages +2021-05-27 21:51:08 [INF] API Security: POST http://target:7777/api/messages +2021-05-27 21:51:08 [INF] API Security: ------------------------------------------------ +``` + +#### Examples + +This example excludes the `/auth` resource. This does not exclude child resources (`/auth/child`). + +```yaml +variables: + DAST_API_EXCLUDE_PATHS=/auth +``` + +To exclude `/auth`, and child resources (`/auth/child`), we use a wildcard. + +```yaml +variables: + DAST_API_EXCLUDE_PATHS=/auth* +``` + +To exclude multiple paths we use the `;` character. In this example we exclude `/auth*` and `/v1/*`. + +```yaml +variables: + DAST_API_EXCLUDE_PATHS=/auth*;/v1/* +``` + ## Running your first scan When configured correctly, a CI/CD pipeline contains a `dast` stage and an `dast_api` job. The job only fails when an invalid configuration is provided. During normal operation, the job always succeeds even if vulnerabilities are identified during testing. @@ -1089,7 +1131,7 @@ The best-suited solution will depend on whether or not your target API changes f #### Static environment solution -This solution is for pipelines in which the target API URL doesn't change (is static). +This solution is for pipelines in which the target API URL doesn't change (is static). **Add environmental variable** @@ -1106,7 +1148,7 @@ include: #### Dynamic environment solutions -In a dynamic environment your target API changes for each different deployment. In this case, there is more than one possible solution, we recommend you use the `environment_url.txt` file when dealing with dynamic environments. +In a dynamic environment your target API changes for each different deployment. In this case, there is more than one possible solution, we recommend you use the `environment_url.txt` file when dealing with dynamic environments. **Use environment_url.txt** diff --git a/lib/gitlab/health_checks/redis/redis_check.rb b/lib/gitlab/health_checks/redis/redis_check.rb index f7e46fce134..44b85bf886e 100644 --- a/lib/gitlab/health_checks/redis/redis_check.rb +++ b/lib/gitlab/health_checks/redis/redis_check.rb @@ -20,7 +20,8 @@ module Gitlab def check ::Gitlab::HealthChecks::Redis::CacheCheck.check_up && ::Gitlab::HealthChecks::Redis::QueuesCheck.check_up && - ::Gitlab::HealthChecks::Redis::SharedStateCheck.check_up + ::Gitlab::HealthChecks::Redis::SharedStateCheck.check_up && + ::Gitlab::HealthChecks::Redis::TraceChunksCheck.check_up end end end diff --git a/lib/gitlab/health_checks/redis/trace_chunks_check.rb b/lib/gitlab/health_checks/redis/trace_chunks_check.rb new file mode 100644 index 00000000000..cf9fa700b0a --- /dev/null +++ b/lib/gitlab/health_checks/redis/trace_chunks_check.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module HealthChecks + module Redis + class TraceChunksCheck + extend SimpleAbstractCheck + + class << self + def check_up + check + end + + private + + def metric_prefix + 'redis_trace_chunks_ping' + end + + def successful?(result) + result == 'PONG' + end + + # rubocop: disable CodeReuse/ActiveRecord + def check + catch_timeout 10.seconds do + Gitlab::Redis::TraceChunks.with(&:ping) + end + end + # rubocop: enable CodeReuse/ActiveRecord + end + end + end + end +end diff --git a/lib/gitlab/instrumentation/redis.rb b/lib/gitlab/instrumentation/redis.rb index 9a9d3a866b1..ab0e56adc32 100644 --- a/lib/gitlab/instrumentation/redis.rb +++ b/lib/gitlab/instrumentation/redis.rb @@ -8,8 +8,9 @@ module Gitlab Cache = Class.new(RedisBase).enable_redis_cluster_validation Queues = Class.new(RedisBase) SharedState = Class.new(RedisBase).enable_redis_cluster_validation + TraceChunks = Class.new(RedisBase).enable_redis_cluster_validation - STORAGES = [ActionCable, Cache, Queues, SharedState].freeze + STORAGES = [ActionCable, Cache, Queues, SharedState, TraceChunks].freeze # Milliseconds represented in seconds (from 1 millisecond to 2 seconds). QUERY_TIME_BUCKETS = [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2].freeze diff --git a/lib/gitlab/redis/trace_chunks.rb b/lib/gitlab/redis/trace_chunks.rb new file mode 100644 index 00000000000..a2e77cb5df5 --- /dev/null +++ b/lib/gitlab/redis/trace_chunks.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module Redis + class TraceChunks < ::Gitlab::Redis::Wrapper + # The data we store on TraceChunks used to be stored on SharedState. + def self.config_fallback + SharedState + end + end + end +end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 32447d39c02..fa4e7f387ed 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -64,8 +64,19 @@ module Gitlab def config_file_name [ + # Instance specific config sources: ENV["GITLAB_REDIS_#{store_name.underscore.upcase}_CONFIG_FILE"], config_file_path("redis.#{store_name.underscore}.yml"), + + # The current Redis instance may have been split off from another one + # (e.g. TraceChunks was split off from SharedState). There are + # installations out there where the lowest priority config source + # (resque.yml) contains bogus values. In those cases, config_file_name + # should resolve to the instance we originated from (the + # "config_fallback") rather than resque.yml. + config_fallback&.config_file_name, + + # Global config sources: ENV['GITLAB_REDIS_CONFIG_FILE'], config_file_path('resque.yml') ].compact.first @@ -75,6 +86,10 @@ module Gitlab name.demodulize end + def config_fallback + nil + end + def instrumentation_class "::Gitlab::Instrumentation::Redis::#{store_name}".constantize end diff --git a/scripts/prepare_build.sh b/scripts/prepare_build.sh index 5753a0af4f8..e0be80d429f 100644 --- a/scripts/prepare_build.sh +++ b/scripts/prepare_build.sh @@ -42,6 +42,9 @@ sed -i 's|url:.*$|url: redis://redis:6379/11|g' config/redis.queues.yml cp config/redis.shared_state.yml.example config/redis.shared_state.yml sed -i 's|url:.*$|url: redis://redis:6379/12|g' config/redis.shared_state.yml +cp config/redis.trace_chunks.yml.example config/redis.trace_chunks.yml +sed -i 's|url:.*$|url: redis://redis:6379/13|g' config/redis.trace_chunks.yml + if [ "$SETUP_DB" != "false" ]; then setup_db elif getent hosts postgres; then diff --git a/spec/lib/gitlab/health_checks/redis/trace_chunks_check_spec.rb b/spec/lib/gitlab/health_checks/redis/trace_chunks_check_spec.rb new file mode 100644 index 00000000000..5fb5232a4dd --- /dev/null +++ b/spec/lib/gitlab/health_checks/redis/trace_chunks_check_spec.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_relative '../simple_check_shared' + +RSpec.describe Gitlab::HealthChecks::Redis::TraceChunksCheck do + include_examples 'simple_check', 'redis_trace_chunks_ping', 'RedisTraceChunks', 'PONG' +end diff --git a/spec/lib/gitlab/instrumentation/redis_spec.rb b/spec/lib/gitlab/instrumentation/redis_spec.rb index d712d09bdd8..6cddf958f2a 100644 --- a/spec/lib/gitlab/instrumentation/redis_spec.rb +++ b/spec/lib/gitlab/instrumentation/redis_spec.rb @@ -68,7 +68,8 @@ RSpec.describe Gitlab::Instrumentation::Redis do .to contain_exactly(details_row.merge(storage: 'ActionCable'), details_row.merge(storage: 'Cache'), details_row.merge(storage: 'Queues'), - details_row.merge(storage: 'SharedState')) + details_row.merge(storage: 'SharedState'), + details_row.merge(storage: 'TraceChunks')) end end end diff --git a/spec/lib/gitlab/redis/trace_chunks_spec.rb b/spec/lib/gitlab/redis/trace_chunks_spec.rb new file mode 100644 index 00000000000..e974dc519d6 --- /dev/null +++ b/spec/lib/gitlab/redis/trace_chunks_spec.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Redis::TraceChunks do + let(:instance_specific_config_file) { "config/redis.trace_chunks.yml" } + let(:environment_config_file_name) { "GITLAB_REDIS_TRACE_CHUNKS_CONFIG_FILE" } + let(:shared_state_config_file) { nil } + + before do + allow(Gitlab::Redis::SharedState).to receive(:config_file_name).and_return(shared_state_config_file) + end + + include_examples "redis_shared_examples" + + describe '.config_file_name' do + subject { described_class.config_file_name } + + let(:rails_root) { Dir.mktmpdir('redis_shared_examples') } + + before do + # Undo top-level stub of config_file_name because we are testing that method now. + allow(described_class).to receive(:config_file_name).and_call_original + + allow(described_class).to receive(:rails_root).and_return(rails_root) + FileUtils.mkdir_p(File.join(rails_root, 'config')) + end + + after do + FileUtils.rm_rf(rails_root) + end + + context 'when there is only a resque.yml' do + before do + FileUtils.touch(File.join(rails_root, 'config/resque.yml')) + end + + it { expect(subject).to eq("#{rails_root}/config/resque.yml") } + + context 'and there is a global env override' do + before do + stub_env('GITLAB_REDIS_CONFIG_FILE', 'global override') + end + + it { expect(subject).to eq('global override') } + + context 'and SharedState has a different config file' do + let(:shared_state_config_file) { 'shared state config file' } + + it { expect(subject).to eq('shared state config file') } + end + end + end + end +end