diff --git a/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/listener.rb b/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/listener.rb
index 6f4dce9df33fcc255c5c7dd6854751e0775e8312..c40d9e90b56ce1a37f0bfac7e07545b6ee1cc9a9 100644
--- a/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/listener.rb
+++ b/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/listener.rb
@@ -36,6 +36,10 @@ def example_passed(notification)
       end
 
       def dump_summary(_)
+        rails_logger_warn(
+          "\n#{flaky_examples.count} known flaky example(s) detected. " \
+          "Writing this to #{Config.flaky_examples_report_path}.\n"
+        )
         Report.new(flaky_examples).write(Config.flaky_examples_report_path)
 
         return unless new_flaky_examples.any?
diff --git a/gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/listener_spec.rb b/gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/listener_spec.rb
index b46044e45218ce531c3bcaf2dc103d97e24910b8..a830226d88f48acc25772ad142c8a46992d77bfd 100644
--- a/gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/listener_spec.rb
+++ b/gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/listener_spec.rb
@@ -197,6 +197,8 @@
   end
 
   describe '#dump_summary' do
+    subject { listener.dump_summary(nil) }
+
     let(:listener) { described_class.new(suite_flaky_example_report.to_json) }
     let(:new_flaky_rspec_example) { double(new_example_attrs.merge(attempts: 2)) }
     let(:already_flaky_rspec_example) { double(already_flaky_example_attrs.merge(attempts: 2)) }
@@ -207,6 +209,39 @@
       allow(Kernel).to receive(:warn)
     end
 
+    context 'when not flaky tests were found' do
+      it 'prints a message in the console' do
+        allow(Kernel).to receive(:warn).and_call_original
+
+        expect { subject }.to output(
+          %r{0 known flaky example\(s\) detected\. Writing this to rspec/flaky/report\.json}
+        ).to_stderr
+      end
+    end
+
+    context 'when existing flaky tests were found' do
+      before do
+        listener.example_passed(notification_already_flaky_rspec_example)
+      end
+
+      it 'does write them in the correct report' do
+        report = double
+
+        expect(Gitlab::RspecFlaky::Report).to receive(:new).with(listener.flaky_examples).and_return(report)
+        expect(report).to receive(:write).with(Gitlab::RspecFlaky::Config.flaky_examples_report_path)
+
+        subject
+      end
+
+      it 'prints a message in the console' do
+        allow(Kernel).to receive(:warn).and_call_original
+
+        expect { subject }.to output(
+          %r{1 known flaky example\(s\) detected\. Writing this to rspec/flaky/report\.json}
+        ).to_stderr
+      end
+    end
+
     context 'when a report file path is set by FLAKY_RSPEC_REPORT_PATH' do
       it 'delegates the writes to RspecFlaky::Report' do
         listener.example_passed(notification_new_flaky_rspec_example)
@@ -222,7 +257,7 @@
           .to receive(:new).with(listener.__send__(:new_flaky_examples)).and_return(report2)
         expect(report2).to receive(:write).with(Gitlab::RspecFlaky::Config.new_flaky_examples_report_path)
 
-        listener.dump_summary(nil)
+        subject
       end
     end
   end