diff --git a/app/controllers/chaos_controller.rb b/app/controllers/chaos_controller.rb index 0ec6a2cb38aff6958cf8d9340a65e78e57dc60fd..1cfcd2905f211c07299d54a3e854bd147d90b48e 100644 --- a/app/controllers/chaos_controller.rb +++ b/app/controllers/chaos_controller.rb @@ -20,7 +20,11 @@ def sleep end def kill - do_chaos :kill, Chaos::KillWorker + do_chaos :kill, Chaos::KillWorker, 'KILL' + end + + def quit + do_chaos :kill, Chaos::KillWorker, 'QUIT' end def gc diff --git a/app/workers/chaos/kill_worker.rb b/app/workers/chaos/kill_worker.rb index 3dedd47a1f9b99b5c59a581b7494b39bc19d08bb..4148c139d420aaca41fc997727632d53022d1a64 100644 --- a/app/workers/chaos/kill_worker.rb +++ b/app/workers/chaos/kill_worker.rb @@ -7,8 +7,8 @@ class KillWorker # rubocop:disable Scalability/IdempotentWorker sidekiq_options retry: false - def perform - Gitlab::Chaos.kill + def perform(signal) + Gitlab::Chaos.kill(signal) end end end diff --git a/changelogs/unreleased/mk-chaos-quit.yml b/changelogs/unreleased/mk-chaos-quit.yml new file mode 100644 index 0000000000000000000000000000000000000000..3db35a1015959c8e413b55742fd303e7f2a59ba1 --- /dev/null +++ b/changelogs/unreleased/mk-chaos-quit.yml @@ -0,0 +1,5 @@ +--- +title: Add a chaos endpoint that signals QUIT +merge_request: 58755 +author: +type: changed diff --git a/config/routes.rb b/config/routes.rb index c21c920117f7c732e918580d954720aae9dbdee6..1258675df86931192b98489efbf27666459af1ab 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -179,6 +179,7 @@ get :db_spin get :sleep get :kill + get :quit post :gc end end diff --git a/doc/development/chaos_endpoints.md b/doc/development/chaos_endpoints.md index 85c93f521ac4dd37062f13496917ced63d06bbd4..56e91acbc4ac7cde3efd0e356806f528fd497974 100644 --- a/doc/development/chaos_endpoints.md +++ b/doc/development/chaos_endpoints.md @@ -146,10 +146,10 @@ curl "http://localhost:3000/-/chaos/sleep?duration_s=60&token=secret" ## Kill -This endpoint simulates the unexpected death of a worker process using a `kill` signal. +This endpoint simulates the unexpected death of a worker process using the `KILL` signal. -Because this endpoint uses the `KILL` signal, the worker isn't given an -opportunity to cleanup or shutdown. +Because this endpoint uses the `KILL` signal, the process isn't given an +opportunity to clean up or shut down. ```plaintext GET /-/chaos/kill @@ -158,13 +158,33 @@ GET /-/chaos/kill?async=true | Attribute | Type | Required | Description | | ------------ | ------- | -------- | ---------------------------------------------------------------------- | -| `async` | boolean | no | Set to true to kill a Sidekiq background worker process | +| `async` | boolean | no | Set to true to signal a Sidekiq background worker process | ```shell curl "http://localhost:3000/-/chaos/kill" --header 'X-Chaos-Secret: secret' curl "http://localhost:3000/-/chaos/kill?token=secret" ``` +## Quit + +This endpoint simulates the unexpected death of a worker process using the `QUIT` signal. +Unlike `KILL`, the `QUIT` signal will also attempt to write a core dump. +See [core(5)](https://man7.org/linux/man-pages/man5/core.5.html) for more information. + +```plaintext +GET /-/chaos/quit +GET /-/chaos/quit?async=true +``` + +| Attribute | Type | Required | Description | +| ------------ | ------- | -------- | ---------------------------------------------------------------------- | +| `async` | boolean | no | Set to true to signal a Sidekiq background worker process | + +```shell +curl "http://localhost:3000/-/chaos/quit" --header 'X-Chaos-Secret: secret' +curl "http://localhost:3000/-/chaos/quit?token=secret" +``` + ## Run garbage collector This endpoint triggers a GC run on the worker handling the request and returns its worker ID diff --git a/lib/gitlab/chaos.rb b/lib/gitlab/chaos.rb index 029a9210dc99a7452829399d76c47454cff5f948..495f12882e5f93b99b4ca8931aba78ed6b4d978d 100644 --- a/lib/gitlab/chaos.rb +++ b/lib/gitlab/chaos.rb @@ -43,9 +43,9 @@ def self.sleep(duration_s) Kernel.sleep(duration_s) end - # Kill will send a SIGKILL signal to the current process - def self.kill - Process.kill("KILL", Process.pid) + # Kill will send the given signal to the current process. + def self.kill(signal) + Process.kill(signal, Process.pid) end def self.run_gc diff --git a/spec/controllers/chaos_controller_spec.rb b/spec/controllers/chaos_controller_spec.rb index cb4f12ff8296a8cea606c11aa07aea1282e74b81..26ae4a6b693a9a79926217498bde4607995022fe 100644 --- a/spec/controllers/chaos_controller_spec.rb +++ b/spec/controllers/chaos_controller_spec.rb @@ -109,7 +109,7 @@ describe '#kill' do it 'calls synchronously' do - expect(Gitlab::Chaos).to receive(:kill).with(no_args) + expect(Gitlab::Chaos).to receive(:kill).with('KILL') get :kill @@ -117,7 +117,7 @@ end it 'calls asynchronously' do - expect(Chaos::KillWorker).to receive(:perform_async).with(no_args) + expect(Chaos::KillWorker).to receive(:perform_async).with('KILL') get :kill, params: { async: 1 } @@ -125,6 +125,24 @@ end end + describe '#quit' do + it 'calls synchronously' do + expect(Gitlab::Chaos).to receive(:kill).with('QUIT') + + get :quit + + expect(response).to have_gitlab_http_status(:ok) + end + + it 'calls asynchronously' do + expect(Chaos::KillWorker).to receive(:perform_async).with('QUIT') + + get :quit, params: { async: 1 } + + expect(response).to have_gitlab_http_status(:ok) + end + end + describe '#gc' do let(:gc_stat) { GC.stat.stringify_keys }