From cfe3cfb370ca112c5cab2af3550cf68c2ec6042d Mon Sep 17 00:00:00 2001 From: Andrew Newdigate Date: Thu, 1 Nov 2018 12:56:32 +0000 Subject: [PATCH 1/6] Adding chaos to GitLab through chaos endpoints --- app/controllers/chaos_controller.rb | 9 +++++++++ config/routes.rb | 2 ++ 2 files changed, 11 insertions(+) create mode 100644 app/controllers/chaos_controller.rb diff --git a/app/controllers/chaos_controller.rb b/app/controllers/chaos_controller.rb new file mode 100644 index 00000000000..ff0ec75f39a --- /dev/null +++ b/app/controllers/chaos_controller.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class ChaosController < ActionController::Base + def sleep + duration_s = params[:duration_s] ? params[:duration_s].to_i : 30 + Kernel.sleep duration_s + render text: "OK", content_type: 'text/plain' + end +end diff --git a/config/routes.rb b/config/routes.rb index 37c7f98ec98..4764b85cc30 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -82,6 +82,8 @@ Rails.application.routes.draw do draw :operations draw :instance_statistics + + get '/chaos/sleep' => 'chaos#sleep' end draw :api From 83dc8f1c666419434a23e467508061b6897fdfb6 Mon Sep 17 00:00:00 2001 From: Andrew Newdigate Date: Thu, 1 Nov 2018 13:16:21 +0000 Subject: [PATCH 2/6] Leak memory, spin cpu and kill the process --- app/controllers/chaos_controller.rb | 23 +++++++++++++++++++++++ config/routes.rb | 3 +++ 2 files changed, 26 insertions(+) diff --git a/app/controllers/chaos_controller.rb b/app/controllers/chaos_controller.rb index ff0ec75f39a..bdb99995532 100644 --- a/app/controllers/chaos_controller.rb +++ b/app/controllers/chaos_controller.rb @@ -1,9 +1,32 @@ # frozen_string_literal: true class ChaosController < ActionController::Base + def leakmem + memory_mb = params[:memory_mb] ? params[:memory_mb].to_i : 100 + retainer = [] + + memory_mb.times { retainer << "x" * (1024 * 1024) } + render text: "OK", content_type: 'text/plain' + end + + def cpuspin + duration_s = params[:duration_s] ? params[:duration_s].to_i : 30 + end_time = Time.now + duration_s.seconds; + while Time.now < end_time + 10_000.times { } + end + + render text: "OK", content_type: 'text/plain' + end + def sleep duration_s = params[:duration_s] ? params[:duration_s].to_i : 30 Kernel.sleep duration_s render text: "OK", content_type: 'text/plain' end + + def kill + Process.kill("KILL", Process.pid) + end + end diff --git a/config/routes.rb b/config/routes.rb index 4764b85cc30..d214356f3e9 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -83,7 +83,10 @@ Rails.application.routes.draw do draw :operations draw :instance_statistics + get '/chaos/leakmem' => 'chaos#leakmem' + get '/chaos/cpuspin' => 'chaos#cpuspin' get '/chaos/sleep' => 'chaos#sleep' + get '/chaos/kill' => 'chaos#kill' end draw :api From 847c81b755b6b4146eb3fa3d5912f3d573739bd1 Mon Sep 17 00:00:00 2001 From: Andrew Newdigate Date: Thu, 1 Nov 2018 17:20:34 +0000 Subject: [PATCH 3/6] Add documentation, secure routes, etc --- app/controllers/chaos_controller.rb | 13 +++++ config/routes.rb | 10 ++-- doc/development/chaos_endpoints.md | 83 +++++++++++++++++++++++++++++ doc/development/performance.md | 1 + 4 files changed, 103 insertions(+), 4 deletions(-) create mode 100644 doc/development/chaos_endpoints.md diff --git a/app/controllers/chaos_controller.rb b/app/controllers/chaos_controller.rb index bdb99995532..6593b748130 100644 --- a/app/controllers/chaos_controller.rb +++ b/app/controllers/chaos_controller.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class ChaosController < ActionController::Base + before_action :validate_request + def leakmem memory_mb = params[:memory_mb] ? params[:memory_mb].to_i : 100 retainer = [] @@ -29,4 +31,15 @@ class ChaosController < ActionController::Base Process.kill("KILL", Process.pid) end + private + + def validate_request + secret = ENV['GITLAB_CHAOS_SECRET'] + return unless secret + + unless request.headers["HTTP_X_CHAOS_SECRET"] == secret + render text: "To experience chaos, please set X-Chaos-Secret header", content_type: 'text/plain', status: 401 + end + end + end diff --git a/config/routes.rb b/config/routes.rb index d214356f3e9..d4c19a03ff8 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -83,10 +83,12 @@ Rails.application.routes.draw do draw :operations draw :instance_statistics - get '/chaos/leakmem' => 'chaos#leakmem' - get '/chaos/cpuspin' => 'chaos#cpuspin' - get '/chaos/sleep' => 'chaos#sleep' - get '/chaos/kill' => 'chaos#kill' + if ENV['GITLAB_ENABLE_CHAOS_ENDPOINTS'] + get '/chaos/leakmem' => 'chaos#leakmem' + get '/chaos/cpuspin' => 'chaos#cpuspin' + get '/chaos/sleep' => 'chaos#sleep' + get '/chaos/kill' => 'chaos#kill' + end end draw :api diff --git a/doc/development/chaos_endpoints.md b/doc/development/chaos_endpoints.md new file mode 100644 index 00000000000..4546d1498c0 --- /dev/null +++ b/doc/development/chaos_endpoints.md @@ -0,0 +1,83 @@ +# Generating Chaos in a test GitLab instance + +As [Werner Vogels](https://twitter.com/Werner), the CTO at Amazon Web Services, famously put it, **Everything fails, all the time**. + +As a developer, it's as important to consider the failure modes in which your software will operate as much as normal operation. Doing so can mean the difference between a minor hiccup leading to a scattering of 500 errors experienced by a tiny fraction of users and a full site outage affect all users for an extended period. + +To paraphrase [Tolstoy](https://en.wikipedia.org/wiki/Anna_Karenina_principle), _all happy servers are alike, but all failing servers are failing in their own way_. Luckily, there are ways we can attempt to simulate these failure modes, and the chaos endpoints are tools for assisting in this process. + +Currently, there are four endpoints for simulating the following conditions: slow requests, cpu-bound requests, memory leaks and unexpected process crashes. + +## Enabling Chaos Endpoints + +For obvious reasons, these endpoints are not enabled by default. They can be enabled by setting the `GITLAB_ENABLE_CHAOS_ENDPOINTS` environment variable. + +For example, if you're using the [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit) this can be done with the following command: + +```shell +GITLAB_ENABLE_CHAOS_ENDPOINTS=1 gdk run +``` + +### Securing the Chaos Endpoints + +**It is highly recommended that you secure access to the Chaos endpoints using a secret token**. This is recommended when enabling these endpoints locally, and essential when running in a staging or other shared environment. _It goes without saying that you should not enable them in production unless you absolutely know what you're doing._ + +A secret can be set through the `GITLAB_CHAOS_SECRET` environment variable. For example, when using the [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit) this can be done with the following command line: + +```shell +GITLAB_ENABLE_CHAOS_ENDPOINTS=1 GITLAB_CHAOS_SECRET=secret gdk run +``` + +Replace `secret` with your own secret token. + +## Invoking Chaos + +Once you have enabled the chaos endpoints and restarted the application you can start testing using the endpoints. + +### Memory Leaks + +To simulate a memory leak in your application, use the `/-/chaos/leakmem` endpoint. + +For example, if your GitLab instance is listening at `localhost:3000`, you could `curl` the endpoint as follows: + +```shell +curl http://localhost:3000/-/chaos/leakmem?memory_mb=1024 -H 'X-Chaos-Secret: secret' +``` + +The `memory_mb` parameter tells the application how much memory it should leak. + +Note: the memory is not retained after the request, so once its completed, the Ruby garbage collector will attempt to recover the memory. + +### CPU Spin + +This endpoint attempts to fully utilise a single core, at 100%, for the given period. + +```shell +curl http://localhost:3000/-/chaos/cpuspin?duration_s=60 -H 'X-Chaos-Secret: secret' +``` + +The `duration_s` parameter will configure how long the core is utilised. + +Depending on your rack server setup, your request may timeout after a predermined period (normally 60 seconds). If you're using Unicorn, this is done by killing the worker process. + +### Sleep + +This endpoint is similar to the CPU Spin endpoint but simulates off-processor activity, such backend services of IO. It will sleep for a given duration. + +```shell +curl http://localhost:3000/-/chaos/sleep?duration_s=60 -H 'X-Chaos-Secret: secret' +``` + +The `duration_s` parameter will configure how long the request will sleep for. + +As with the CPU Spin endpoint, this may lead to your request timing out if duration exceeds the configured limit. + +### Kill + +This endpoint will simulate the unexpected death of a worker process using a `kill` signal. + +```shell +curl http://localhost:3000/-/chaos/kill -H 'X-Chaos-Secret: secret' +``` + +Note: since this endpoint uses the `KILL` signal, the worker is not given a chance to cleanup or shutdown. diff --git a/doc/development/performance.md b/doc/development/performance.md index c7b10dfd5ce..ec1ac2d49da 100644 --- a/doc/development/performance.md +++ b/doc/development/performance.md @@ -41,6 +41,7 @@ GitLab provides built-in tools to aid the process of improving performance: * [GitLab Performance Monitoring](../administration/monitoring/performance/index.md) * [Request Profiling](../administration/monitoring/performance/request_profiling.md) * [QueryRecoder](query_recorder.md) for preventing `N+1` regressions +* [Chaos Endpoints](chaos_endpoints.md) less for performance, more for availability: tools for testing failure scenarios GitLab employees can use GitLab.com's performance monitoring systems located at , this requires you to log in using your From 04efa0b512953d90e125850146759d09fac2d9bc Mon Sep 17 00:00:00 2001 From: Andrew Newdigate Date: Thu, 1 Nov 2018 18:06:25 +0000 Subject: [PATCH 4/6] Fixing the broken build with style fixes --- app/controllers/chaos_controller.rb | 7 ++----- changelogs/unreleased/52767-more-chaos-for-gitlab.yml | 5 +++++ doc/development/chaos_endpoints.md | 8 ++++---- 3 files changed, 11 insertions(+), 9 deletions(-) create mode 100644 changelogs/unreleased/52767-more-chaos-for-gitlab.yml diff --git a/app/controllers/chaos_controller.rb b/app/controllers/chaos_controller.rb index 6593b748130..392814b4275 100644 --- a/app/controllers/chaos_controller.rb +++ b/app/controllers/chaos_controller.rb @@ -13,10 +13,8 @@ class ChaosController < ActionController::Base def cpuspin duration_s = params[:duration_s] ? params[:duration_s].to_i : 30 - end_time = Time.now + duration_s.seconds; - while Time.now < end_time - 10_000.times { } - end + end_time = Time.now + duration_s.seconds + 10_000.times { } while Time.now < end_time render text: "OK", content_type: 'text/plain' end @@ -41,5 +39,4 @@ class ChaosController < ActionController::Base render text: "To experience chaos, please set X-Chaos-Secret header", content_type: 'text/plain', status: 401 end end - end diff --git a/changelogs/unreleased/52767-more-chaos-for-gitlab.yml b/changelogs/unreleased/52767-more-chaos-for-gitlab.yml new file mode 100644 index 00000000000..067777cb7fa --- /dev/null +++ b/changelogs/unreleased/52767-more-chaos-for-gitlab.yml @@ -0,0 +1,5 @@ +--- +title: Add endpoints for simulating certain failure modes in the application +merge_request: 22746 +author: +type: other diff --git a/doc/development/chaos_endpoints.md b/doc/development/chaos_endpoints.md index 4546d1498c0..318a3270675 100644 --- a/doc/development/chaos_endpoints.md +++ b/doc/development/chaos_endpoints.md @@ -41,7 +41,7 @@ To simulate a memory leak in your application, use the `/-/chaos/leakmem` endpoi For example, if your GitLab instance is listening at `localhost:3000`, you could `curl` the endpoint as follows: ```shell -curl http://localhost:3000/-/chaos/leakmem?memory_mb=1024 -H 'X-Chaos-Secret: secret' +curl http://localhost:3000/-/chaos/leakmem?memory_mb=1024 --header 'X-Chaos-Secret: secret' ``` The `memory_mb` parameter tells the application how much memory it should leak. @@ -53,7 +53,7 @@ Note: the memory is not retained after the request, so once its completed, the R This endpoint attempts to fully utilise a single core, at 100%, for the given period. ```shell -curl http://localhost:3000/-/chaos/cpuspin?duration_s=60 -H 'X-Chaos-Secret: secret' +curl http://localhost:3000/-/chaos/cpuspin?duration_s=60 --header 'X-Chaos-Secret: secret' ``` The `duration_s` parameter will configure how long the core is utilised. @@ -65,7 +65,7 @@ Depending on your rack server setup, your request may timeout after a predermine This endpoint is similar to the CPU Spin endpoint but simulates off-processor activity, such backend services of IO. It will sleep for a given duration. ```shell -curl http://localhost:3000/-/chaos/sleep?duration_s=60 -H 'X-Chaos-Secret: secret' +curl http://localhost:3000/-/chaos/sleep?duration_s=60 --header 'X-Chaos-Secret: secret' ``` The `duration_s` parameter will configure how long the request will sleep for. @@ -77,7 +77,7 @@ As with the CPU Spin endpoint, this may lead to your request timing out if durat This endpoint will simulate the unexpected death of a worker process using a `kill` signal. ```shell -curl http://localhost:3000/-/chaos/kill -H 'X-Chaos-Secret: secret' +curl http://localhost:3000/-/chaos/kill --header 'X-Chaos-Secret: secret' ``` Note: since this endpoint uses the `KILL` signal, the worker is not given a chance to cleanup or shutdown. From f793dad78be67a155d7ce942e7947db39caebaaf Mon Sep 17 00:00:00 2001 From: Andrew Newdigate Date: Mon, 5 Nov 2018 15:34:38 +0000 Subject: [PATCH 5/6] Updated with Sean's feedback --- app/controllers/chaos_controller.rb | 26 ++++++++++++++++++++------ doc/development/chaos_endpoints.md | 7 ++++--- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/app/controllers/chaos_controller.rb b/app/controllers/chaos_controller.rb index 392814b4275..b4f46cddbe9 100644 --- a/app/controllers/chaos_controller.rb +++ b/app/controllers/chaos_controller.rb @@ -4,24 +4,33 @@ class ChaosController < ActionController::Base before_action :validate_request def leakmem - memory_mb = params[:memory_mb] ? params[:memory_mb].to_i : 100 - retainer = [] + memory_mb = (params[:memory_mb]&.to_i || 100) + duration_s = (params[:duration_s]&.to_i || 30).seconds + + start = Time.now + retainer = [] + # Add `n` 1mb chunks of memory to the retainer array + memory_mb.times { retainer << "x" * 1.megabyte } + + duration_taken = (Time.now - start).seconds + Kernel.sleep duration_s - duration_taken if duration_s > duration_taken - memory_mb.times { retainer << "x" * (1024 * 1024) } render text: "OK", content_type: 'text/plain' end def cpuspin - duration_s = params[:duration_s] ? params[:duration_s].to_i : 30 + duration_s = (params[:duration_s]&.to_i || 30).seconds end_time = Time.now + duration_s.seconds - 10_000.times { } while Time.now < end_time + + rand while Time.now < end_time render text: "OK", content_type: 'text/plain' end def sleep - duration_s = params[:duration_s] ? params[:duration_s].to_i : 30 + duration_s = (params[:duration_s]&.to_i || 30).seconds Kernel.sleep duration_s + render text: "OK", content_type: 'text/plain' end @@ -33,6 +42,11 @@ class ChaosController < ActionController::Base def validate_request secret = ENV['GITLAB_CHAOS_SECRET'] + # GITLAB_CHAOS_SECRET is required unless you're running in Development mode + if !secret && !Rails.env.development? + render text: "chaos misconfigured: please configure GITLAB_CHAOS_SECRET when using GITLAB_ENABLE_CHAOS_ENDPOINTS outside of a development environment", content_type: 'text/plain', status: 500 + end + return unless secret unless request.headers["HTTP_X_CHAOS_SECRET"] == secret diff --git a/doc/development/chaos_endpoints.md b/doc/development/chaos_endpoints.md index 318a3270675..71de9bdce50 100644 --- a/doc/development/chaos_endpoints.md +++ b/doc/development/chaos_endpoints.md @@ -41,12 +41,13 @@ To simulate a memory leak in your application, use the `/-/chaos/leakmem` endpoi For example, if your GitLab instance is listening at `localhost:3000`, you could `curl` the endpoint as follows: ```shell -curl http://localhost:3000/-/chaos/leakmem?memory_mb=1024 --header 'X-Chaos-Secret: secret' +curl http://localhost:3000/-/chaos/leakmem?memory_mb=1024&duration_s=10 --header 'X-Chaos-Secret: secret' ``` -The `memory_mb` parameter tells the application how much memory it should leak. +The `memory_mb` parameter tells the application how much memory it should leak. The `duration_s` parameter will ensure the request retains +the memory for this duration at a minimum (default 30s). -Note: the memory is not retained after the request, so once its completed, the Ruby garbage collector will attempt to recover the memory. +Note: the memory is not retained after the request finishes. Once the request has completed, the Ruby garbage collector will attempt to recover the memory. ### CPU Spin From 673f06253d1e799a8024b18270c1a7279fabe9b8 Mon Sep 17 00:00:00 2001 From: Andrew Newdigate Date: Tue, 6 Nov 2018 13:21:04 +0000 Subject: [PATCH 6/6] Documentation updates as per review [skip ci] --- doc/development/chaos_endpoints.md | 109 +++++++++++++++++++---------- doc/development/performance.md | 4 +- 2 files changed, 73 insertions(+), 40 deletions(-) diff --git a/doc/development/chaos_endpoints.md b/doc/development/chaos_endpoints.md index 71de9bdce50..403a5b21827 100644 --- a/doc/development/chaos_endpoints.md +++ b/doc/development/chaos_endpoints.md @@ -1,84 +1,117 @@ -# Generating Chaos in a test GitLab instance +# Generating chaos in a test GitLab instance As [Werner Vogels](https://twitter.com/Werner), the CTO at Amazon Web Services, famously put it, **Everything fails, all the time**. -As a developer, it's as important to consider the failure modes in which your software will operate as much as normal operation. Doing so can mean the difference between a minor hiccup leading to a scattering of 500 errors experienced by a tiny fraction of users and a full site outage affect all users for an extended period. +As a developer, it's as important to consider the failure modes in which your software will operate as much as normal operation. Doing so can mean the difference between a minor hiccup leading to a scattering of `500` errors experienced by a tiny fraction of users and a full site outage that affects all users for an extended period. To paraphrase [Tolstoy](https://en.wikipedia.org/wiki/Anna_Karenina_principle), _all happy servers are alike, but all failing servers are failing in their own way_. Luckily, there are ways we can attempt to simulate these failure modes, and the chaos endpoints are tools for assisting in this process. -Currently, there are four endpoints for simulating the following conditions: slow requests, cpu-bound requests, memory leaks and unexpected process crashes. +Currently, there are four endpoints for simulating the following conditions: -## Enabling Chaos Endpoints +- Slow requests. +- CPU-bound requests. +- Memory leaks. +- Unexpected process crashes. -For obvious reasons, these endpoints are not enabled by default. They can be enabled by setting the `GITLAB_ENABLE_CHAOS_ENDPOINTS` environment variable. +## Enabling chaos endpoints + +For obvious reasons, these endpoints are not enabled by default. They can be enabled by setting the `GITLAB_ENABLE_CHAOS_ENDPOINTS` environment variable to `1`. For example, if you're using the [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit) this can be done with the following command: -```shell +```bash GITLAB_ENABLE_CHAOS_ENDPOINTS=1 gdk run ``` -### Securing the Chaos Endpoints +## Securing the chaos endpoints -**It is highly recommended that you secure access to the Chaos endpoints using a secret token**. This is recommended when enabling these endpoints locally, and essential when running in a staging or other shared environment. _It goes without saying that you should not enable them in production unless you absolutely know what you're doing._ +DANGER: **Danger:** +It is highly recommended that you secure access to the chaos endpoints using a secret token. This is recommended when enabling these endpoints locally and essential when running in a staging or other shared environment. You should not enable them in production unless you absolutely know what you're doing. -A secret can be set through the `GITLAB_CHAOS_SECRET` environment variable. For example, when using the [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit) this can be done with the following command line: +A secret token can be set through the `GITLAB_CHAOS_SECRET` environment variable. For example, when using the [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit) this can be done with the following command: -```shell +```bash GITLAB_ENABLE_CHAOS_ENDPOINTS=1 GITLAB_CHAOS_SECRET=secret gdk run ``` Replace `secret` with your own secret token. -## Invoking Chaos +## Invoking chaos -Once you have enabled the chaos endpoints and restarted the application you can start testing using the endpoints. +Once you have enabled the chaos endpoints and restarted the application, you can start testing using the endpoints. -### Memory Leaks +## Memory leaks To simulate a memory leak in your application, use the `/-/chaos/leakmem` endpoint. -For example, if your GitLab instance is listening at `localhost:3000`, you could `curl` the endpoint as follows: +NOTE: **Note:** +The memory is not retained after the request finishes. Once the request has completed, the Ruby garbage collector will attempt to recover the memory. -```shell +``` +GET /-/chaos/leakmem +GET /-/chaos/leakmem?memory_mb=1024 +GET /-/chaos/leakmem?memory_mb=1024&duration_s=50 +``` + +| Attribute | Type | Required | Description | +| ------------ | ------- | -------- | ---------------------------------------------------------------------------------- | +| `memory_mb` | integer | no | How much memory, in MB, should be leaked. Defaults to 100MB. | +| `duration_s` | integer | no | Minimum duration, in seconds, that the memory should be retained. Defaults to 30s. | + +```bash curl http://localhost:3000/-/chaos/leakmem?memory_mb=1024&duration_s=10 --header 'X-Chaos-Secret: secret' ``` -The `memory_mb` parameter tells the application how much memory it should leak. The `duration_s` parameter will ensure the request retains -the memory for this duration at a minimum (default 30s). - -Note: the memory is not retained after the request finishes. Once the request has completed, the Ruby garbage collector will attempt to recover the memory. - -### CPU Spin +## CPU spin This endpoint attempts to fully utilise a single core, at 100%, for the given period. -```shell +Depending on your rack server setup, your request may timeout after a predermined period (normally 60 seconds). +If you're using Unicorn, this is done by killing the worker process. + +``` +GET /-/chaos/cpuspin +GET /-/chaos/cpuspin?duration_s=50 +``` + +| Attribute | Type | Required | Description | +| ------------ | ------- | -------- | --------------------------------------------------------------------- | +| `duration_s` | integer | no | Duration, in seconds, that the core will be utilised. Defaults to 30s | + +```bash curl http://localhost:3000/-/chaos/cpuspin?duration_s=60 --header 'X-Chaos-Secret: secret' ``` -The `duration_s` parameter will configure how long the core is utilised. +## Sleep -Depending on your rack server setup, your request may timeout after a predermined period (normally 60 seconds). If you're using Unicorn, this is done by killing the worker process. - -### Sleep - -This endpoint is similar to the CPU Spin endpoint but simulates off-processor activity, such backend services of IO. It will sleep for a given duration. - -```shell -curl http://localhost:3000/-/chaos/sleep?duration_s=60 --header 'X-Chaos-Secret: secret' -``` - -The `duration_s` parameter will configure how long the request will sleep for. +This endpoint is similar to the CPU Spin endpoint but simulates off-processor activity, such as network calls to backend services. It will sleep for a given duration. As with the CPU Spin endpoint, this may lead to your request timing out if duration exceeds the configured limit. -### Kill +``` +GET /-/chaos/sleep +GET /-/chaos/sleep?duration_s=50 +``` + +| Attribute | Type | Required | Description | +| ------------ | ------- | -------- | ---------------------------------------------------------------------- | +| `duration_s` | integer | no | Duration, in seconds, that the request will sleep for. Defaults to 30s | + +```bash +curl http://localhost:3000/-/chaos/sleep?duration_s=60 --header 'X-Chaos-Secret: secret' +``` + +## Kill This endpoint will simulate the unexpected death of a worker process using a `kill` signal. -```shell -curl http://localhost:3000/-/chaos/kill --header 'X-Chaos-Secret: secret' +NOTE: **Note:** +Since this endpoint uses the `KILL` signal, the worker is not given a chance to cleanup or shutdown. + +``` +GET /-/chaos/kill ``` -Note: since this endpoint uses the `KILL` signal, the worker is not given a chance to cleanup or shutdown. +```bash +curl http://localhost:3000/-/chaos/kill --header 'X-Chaos-Secret: secret' +``` diff --git a/doc/development/performance.md b/doc/development/performance.md index ec1ac2d49da..e738f2b4b66 100644 --- a/doc/development/performance.md +++ b/doc/development/performance.md @@ -34,14 +34,14 @@ graphs/dashboards. ## Tooling -GitLab provides built-in tools to aid the process of improving performance: +GitLab provides built-in tools to help improve performance and availability: * [Profiling](profiling.md) * [Sherlock](profiling.md#sherlock) * [GitLab Performance Monitoring](../administration/monitoring/performance/index.md) * [Request Profiling](../administration/monitoring/performance/request_profiling.md) * [QueryRecoder](query_recorder.md) for preventing `N+1` regressions -* [Chaos Endpoints](chaos_endpoints.md) less for performance, more for availability: tools for testing failure scenarios +* [Chaos endpoints](chaos_endpoints.md) for testing failure scenarios. Intended mainly for testing availability. GitLab employees can use GitLab.com's performance monitoring systems located at , this requires you to log in using your