From 12683d667262aace70f06dd9d358a22be5d805ad Mon Sep 17 00:00:00 2001
From: Oswaldo Ferreira <oswaldo@gitlab.com>
Date: Wed, 22 Apr 2020 14:37:16 -0300
Subject: [PATCH] Use reactive caching worker for external services
It uses a specific worker/queue to process the work that
require external services. That's because this kind of
work should not be mixed with other time-sensitive work.
---
app/finders/clusters/knative_services_finder.rb | 1 +
app/finders/projects/serverless/functions_finder.rb | 1 +
app/models/clusters/cluster.rb | 2 ++
app/models/concerns/prometheus_adapter.rb | 1 +
app/models/environment.rb | 1 +
app/models/error_tracking/project_error_tracking_setting.rb | 1 +
app/models/ssh_host_key.rb | 1 +
app/services/grafana/proxy_service.rb | 1 +
app/services/metrics/dashboard/grafana_metric_embed_service.rb | 1 +
app/services/pod_logs/elasticsearch_service.rb | 1 +
app/services/pod_logs/kubernetes_service.rb | 1 +
app/services/prometheus/proxy_service.rb | 1 +
spec/controllers/projects/mirrors_controller_spec.rb | 2 +-
spec/services/grafana/proxy_service_spec.rb | 2 +-
.../metrics/dashboard/grafana_metric_embed_service_spec.rb | 2 +-
spec/services/prometheus/proxy_service_spec.rb | 2 +-
16 files changed, 17 insertions(+), 4 deletions(-)
diff --git a/app/finders/clusters/knative_services_finder.rb b/app/finders/clusters/knative_services_finder.rb
index 71cebe4495e1f..af8c42f672f30 100644
--- a/app/finders/clusters/knative_services_finder.rb
+++ b/app/finders/clusters/knative_services_finder.rb
@@ -11,6 +11,7 @@ class KnativeServicesFinder
}.freeze
self.reactive_cache_key = ->(finder) { finder.model_name }
+ self.reactive_cache_work_type = :external_dependency
self.reactive_cache_worker_finder = ->(_id, *cache_args) { from_cache(*cache_args) }
attr_reader :cluster, :environment
diff --git a/app/finders/projects/serverless/functions_finder.rb b/app/finders/projects/serverless/functions_finder.rb
index 3b4ecbb5387fe..13f84e0e3a5ae 100644
--- a/app/finders/projects/serverless/functions_finder.rb
+++ b/app/finders/projects/serverless/functions_finder.rb
@@ -9,6 +9,7 @@ class FunctionsFinder
attr_reader :project
self.reactive_cache_key = ->(finder) { finder.cache_key }
+ self.reactive_cache_work_type = :external_dependency
self.reactive_cache_worker_finder = ->(_id, *args) { from_cache(*args) }
MAX_CLUSTERS = 10
diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb
index 430a9b3c43e17..78dd876020bc1 100644
--- a/app/models/clusters/cluster.rb
+++ b/app/models/clusters/cluster.rb
@@ -26,6 +26,8 @@ class Cluster < ApplicationRecord
KUBE_INGRESS_BASE_DOMAIN = 'KUBE_INGRESS_BASE_DOMAIN'
APPLICATIONS_ASSOCIATIONS = APPLICATIONS.values.map(&:association_name).freeze
+ self.reactive_cache_work_type = :external_dependency
+
belongs_to :user
belongs_to :management_project, class_name: '::Project', optional: true
diff --git a/app/models/concerns/prometheus_adapter.rb b/app/models/concerns/prometheus_adapter.rb
index abc41a1c4766e..761a151a474e9 100644
--- a/app/models/concerns/prometheus_adapter.rb
+++ b/app/models/concerns/prometheus_adapter.rb
@@ -9,6 +9,7 @@ module PrometheusAdapter
self.reactive_cache_lease_timeout = 30.seconds
self.reactive_cache_refresh_interval = 30.seconds
self.reactive_cache_lifetime = 1.minute
+ self.reactive_cache_work_type = :external_dependency
def prometheus_client
raise NotImplementedError
diff --git a/app/models/environment.rb b/app/models/environment.rb
index b2391f33aca18..248e2716b61f1 100644
--- a/app/models/environment.rb
+++ b/app/models/environment.rb
@@ -8,6 +8,7 @@ class Environment < ApplicationRecord
self.reactive_cache_refresh_interval = 1.minute
self.reactive_cache_lifetime = 55.seconds
self.reactive_cache_hard_limit = 10.megabytes
+ self.reactive_cache_work_type = :external_dependency
belongs_to :project, required: true
diff --git a/app/models/error_tracking/project_error_tracking_setting.rb b/app/models/error_tracking/project_error_tracking_setting.rb
index 133850b6ab61e..fa32c8a54502f 100644
--- a/app/models/error_tracking/project_error_tracking_setting.rb
+++ b/app/models/error_tracking/project_error_tracking_setting.rb
@@ -22,6 +22,7 @@ class ProjectErrorTrackingSetting < ApplicationRecord
}x.freeze
self.reactive_cache_key = ->(setting) { [setting.class.model_name.singular, setting.project_id] }
+ self.reactive_cache_work_type = :external_dependency
belongs_to :project
diff --git a/app/models/ssh_host_key.rb b/app/models/ssh_host_key.rb
index 9bd35d308456b..72690ad7d048b 100644
--- a/app/models/ssh_host_key.rb
+++ b/app/models/ssh_host_key.rb
@@ -24,6 +24,7 @@ def as_json(*)
# This is achieved by making the lifetime shorter than the refresh interval.
self.reactive_cache_refresh_interval = 15.minutes
self.reactive_cache_lifetime = 10.minutes
+ self.reactive_cache_work_type = :external_dependency
def self.find_by(opts = {})
opts = HashWithIndifferentAccess.new(opts)
diff --git a/app/services/grafana/proxy_service.rb b/app/services/grafana/proxy_service.rb
index 74fcdc750b015..ac4c3cc091c9b 100644
--- a/app/services/grafana/proxy_service.rb
+++ b/app/services/grafana/proxy_service.rb
@@ -12,6 +12,7 @@ class ProxyService < BaseService
self.reactive_cache_key = ->(service) { service.cache_key }
self.reactive_cache_lease_timeout = 30.seconds
self.reactive_cache_refresh_interval = 30.seconds
+ self.reactive_cache_work_type = :external_dependency
self.reactive_cache_worker_finder = ->(_id, *args) { from_cache(*args) }
attr_accessor :project, :datasource_id, :proxy_path, :query_params
diff --git a/app/services/metrics/dashboard/grafana_metric_embed_service.rb b/app/services/metrics/dashboard/grafana_metric_embed_service.rb
index d58b80162f51d..5fd1e0dda89d6 100644
--- a/app/services/metrics/dashboard/grafana_metric_embed_service.rb
+++ b/app/services/metrics/dashboard/grafana_metric_embed_service.rb
@@ -18,6 +18,7 @@ class GrafanaMetricEmbedService < ::Metrics::Dashboard::BaseEmbedService
self.reactive_cache_lease_timeout = 30.seconds
self.reactive_cache_refresh_interval = 30.minutes
self.reactive_cache_lifetime = 30.minutes
+ self.reactive_cache_work_type = :external_dependency
self.reactive_cache_worker_finder = ->(_id, *args) { from_cache(*args) }
class << self
diff --git a/app/services/pod_logs/elasticsearch_service.rb b/app/services/pod_logs/elasticsearch_service.rb
index 9a9b453c554f8..a9da477cbc1c0 100644
--- a/app/services/pod_logs/elasticsearch_service.rb
+++ b/app/services/pod_logs/elasticsearch_service.rb
@@ -11,6 +11,7 @@ class ElasticsearchService < PodLogs::BaseService
:pod_logs,
:filter_return_keys
+ self.reactive_cache_work_type = :external_dependency
self.reactive_cache_worker_finder = ->(id, _cache_key, namespace, params) { new(::Clusters::Cluster.find(id), namespace, params: params) }
private
diff --git a/app/services/pod_logs/kubernetes_service.rb b/app/services/pod_logs/kubernetes_service.rb
index 0a8072a9037df..f59fc60113f0a 100644
--- a/app/services/pod_logs/kubernetes_service.rb
+++ b/app/services/pod_logs/kubernetes_service.rb
@@ -17,6 +17,7 @@ class KubernetesService < PodLogs::BaseService
:split_logs,
:filter_return_keys
+ self.reactive_cache_work_type = :external_dependency
self.reactive_cache_worker_finder = ->(id, _cache_key, namespace, params) { new(::Clusters::Cluster.find(id), namespace, params: params) }
private
diff --git a/app/services/prometheus/proxy_service.rb b/app/services/prometheus/proxy_service.rb
index 99c739a630be2..085cfc761964a 100644
--- a/app/services/prometheus/proxy_service.rb
+++ b/app/services/prometheus/proxy_service.rb
@@ -17,6 +17,7 @@ class ProxyService < BaseService
# is expected to change *and* be fetched again by the frontend
self.reactive_cache_refresh_interval = 90.seconds
self.reactive_cache_lifetime = 1.minute
+ self.reactive_cache_work_type = :external_dependency
self.reactive_cache_worker_finder = ->(_id, *args) { from_cache(*args) }
attr_accessor :proxyable, :method, :path, :params
diff --git a/spec/controllers/projects/mirrors_controller_spec.rb b/spec/controllers/projects/mirrors_controller_spec.rb
index faeade0d73798..8cd940978c06b 100644
--- a/spec/controllers/projects/mirrors_controller_spec.rb
+++ b/spec/controllers/projects/mirrors_controller_spec.rb
@@ -189,7 +189,7 @@
context 'no data in cache' do
it 'requests the cache to be filled and returns a 204 response' do
- expect(ReactiveCachingWorker).to receive(:perform_async).with(cache.class, cache.id).at_least(:once)
+ expect(ExternalServiceReactiveCachingWorker).to receive(:perform_async).with(cache.class, cache.id).at_least(:once)
do_get(project)
diff --git a/spec/services/grafana/proxy_service_spec.rb b/spec/services/grafana/proxy_service_spec.rb
index 694d531c9fc86..8cb7210524a19 100644
--- a/spec/services/grafana/proxy_service_spec.rb
+++ b/spec/services/grafana/proxy_service_spec.rb
@@ -66,7 +66,7 @@
context 'with caching', :use_clean_rails_memory_store_caching do
context 'when value not present in cache' do
it 'returns nil' do
- expect(ReactiveCachingWorker)
+ expect(ExternalServiceReactiveCachingWorker)
.to receive(:perform_async)
.with(service.class, service.id, *cache_params)
diff --git a/spec/services/metrics/dashboard/grafana_metric_embed_service_spec.rb b/spec/services/metrics/dashboard/grafana_metric_embed_service_spec.rb
index 034d6aba5d6ce..3eebbe98c9d9d 100644
--- a/spec/services/metrics/dashboard/grafana_metric_embed_service_spec.rb
+++ b/spec/services/metrics/dashboard/grafana_metric_embed_service_spec.rb
@@ -154,7 +154,7 @@
context 'when value not present in cache' do
it 'returns nil' do
- expect(ReactiveCachingWorker)
+ expect(ExternalServiceReactiveCachingWorker)
.to receive(:perform_async)
.with(service.class, service.id, *cache_params)
diff --git a/spec/services/prometheus/proxy_service_spec.rb b/spec/services/prometheus/proxy_service_spec.rb
index 5a036194d01aa..656ccea10def6 100644
--- a/spec/services/prometheus/proxy_service_spec.rb
+++ b/spec/services/prometheus/proxy_service_spec.rb
@@ -117,7 +117,7 @@
context 'when value not present in cache' do
it 'returns nil' do
- expect(ReactiveCachingWorker)
+ expect(ExternalServiceReactiveCachingWorker)
.to receive(:perform_async)
.with(subject.class, subject.id, *opts)
--
GitLab