diff --git a/.flayignore b/.flayignore
index 47597025115c9cd540035b84fe5a859c12c92287..e2d0a2e50c55b58710ac520e1401ef38be798977 100644
--- a/.flayignore
+++ b/.flayignore
@@ -3,3 +3,4 @@ lib/gitlab/sanitizers/svg/whitelist.rb
lib/gitlab/diff/position_tracer.rb
app/policies/project_policy.rb
app/models/concerns/relative_positioning.rb
+lib/gitlab/redis/*.rb
diff --git a/.gitignore b/.gitignore
index 0d6194dd1e5346077955d3ffed3f7f174b21cccb..3baf640a9c36d017cc22fdf0f59185c6de6db3a9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -31,6 +31,9 @@ eslint-report.html
/config/initializers/smtp_settings.rb
/config/initializers/relative_url.rb
/config/resque.yml
+/config/redis.cache.yml
+/config/redis.queues.yml
+/config/redis.shared_state.yml
/config/unicorn.rb
/config/secrets.yml
/config/sidekiq.yml
diff --git a/app/controllers/health_controller.rb b/app/controllers/health_controller.rb
index abc832e6ddc73d87b5cb774801e6aadde8546721..3dbacbbc8974f674b0a79859fa99f7f7835b46eb 100644
--- a/app/controllers/health_controller.rb
+++ b/app/controllers/health_controller.rb
@@ -4,7 +4,10 @@ class HealthController < ActionController::Base
CHECKS = [
Gitlab::HealthChecks::DbCheck,
- Gitlab::HealthChecks::RedisCheck,
+ Gitlab::HealthChecks::Redis::RedisCheck,
+ Gitlab::HealthChecks::Redis::CacheCheck,
+ Gitlab::HealthChecks::Redis::QueuesCheck,
+ Gitlab::HealthChecks::Redis::SharedStateCheck,
Gitlab::HealthChecks::FsShardsCheck
].freeze
diff --git a/app/models/ci/runner.rb b/app/models/ci/runner.rb
index fb0fbb43fb1763b1bd4b0cecf5acff3639690743..c6d23898560be2bbedbc2cd97f2053b56ed94948 100644
--- a/app/models/ci/runner.rb
+++ b/app/models/ci/runner.rb
@@ -149,7 +149,7 @@ def is_runner_queue_value_latest?(value)
private
def cleanup_runner_queue
- Gitlab::Redis.with do |redis|
+ Gitlab::Redis::Queues.with do |redis|
redis.del(runner_queue_key)
end
end
diff --git a/app/models/project.rb b/app/models/project.rb
index d576016466349b0fbd4c878f42e3ab00f9dd3b2b..e50818e3dfffdc707c14568966d7fb2597da008b 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -1386,15 +1386,15 @@ def append_or_update_attribute(name, value)
end
def pushes_since_gc
- Gitlab::Redis.with { |redis| redis.get(pushes_since_gc_redis_key).to_i }
+ Gitlab::Redis::SharedState.with { |redis| redis.get(pushes_since_gc_redis_shared_state_key).to_i }
end
def increment_pushes_since_gc
- Gitlab::Redis.with { |redis| redis.incr(pushes_since_gc_redis_key) }
+ Gitlab::Redis::SharedState.with { |redis| redis.incr(pushes_since_gc_redis_shared_state_key) }
end
def reset_pushes_since_gc
- Gitlab::Redis.with { |redis| redis.del(pushes_since_gc_redis_key) }
+ Gitlab::Redis::SharedState.with { |redis| redis.del(pushes_since_gc_redis_shared_state_key) }
end
def route_map_for(commit_sha)
@@ -1457,7 +1457,7 @@ def cross_project_reference?(from)
from && self != from
end
- def pushes_since_gc_redis_key
+ def pushes_since_gc_redis_shared_state_key
"projects/#{id}/pushes_since_gc"
end
diff --git a/app/services/metrics_service.rb b/app/services/metrics_service.rb
index d726db4e99bcc352c5e38c1ea1fc63d2d553a140..c92f070601cd6678ebeed63b84c245a8e1d3c40f 100644
--- a/app/services/metrics_service.rb
+++ b/app/services/metrics_service.rb
@@ -3,7 +3,10 @@
class MetricsService
CHECKS = [
Gitlab::HealthChecks::DbCheck,
- Gitlab::HealthChecks::RedisCheck,
+ Gitlab::HealthChecks::Redis::RedisCheck,
+ Gitlab::HealthChecks::Redis::CacheCheck,
+ Gitlab::HealthChecks::Redis::QueuesCheck,
+ Gitlab::HealthChecks::Redis::SharedStateCheck,
Gitlab::HealthChecks::FsShardsCheck
].freeze
diff --git a/app/services/milestones/destroy_service.rb b/app/services/milestones/destroy_service.rb
index 600ebcfbecb9db88a469873bb8ac6c241ec1d2dc..e457212508f311cdee24ebe6490082429d9e2d64 100644
--- a/app/services/milestones/destroy_service.rb
+++ b/app/services/milestones/destroy_service.rb
@@ -1,17 +1,15 @@
module Milestones
class DestroyService < Milestones::BaseService
def execute(milestone)
- return unless milestone.is_project_milestone?
-
Milestone.transaction do
update_params = { milestone: nil }
milestone.issues.each do |issue|
- Issues::UpdateService.new(parent, current_user, update_params).execute(issue)
+ Issues::UpdateService.new(project, current_user, update_params).execute(issue)
end
milestone.merge_requests.each do |merge_request|
- MergeRequests::UpdateService.new(parent, current_user, update_params).execute(merge_request)
+ MergeRequests::UpdateService.new(project, current_user, update_params).execute(merge_request)
end
event_service.destroy_milestone(milestone, current_user)
diff --git a/config/README.md b/config/README.md
index 0a5ea2424e0a5b41642417bc2244c62a23dad3d7..2778d0d4f02e88f44b5bd379989bfb973c0ac3ba 100644
--- a/config/README.md
+++ b/config/README.md
@@ -19,4 +19,132 @@ an ERB file and then loads the resulting YML as its configuration.
This file is called `resque.yml` for historical reasons. We are **NOT**
using Resque at the moment. It is used to specify Redis configuration
-values instead.
+values when a single database instance of Redis is desired.
+
+# Advanced Redis configuration files
+
+In more advanced configurations of Redis key-value storage, it is desirable
+to separate the keys by lifecycle and intended use to ease provisioning and
+management of scalable Redis clusters.
+
+These settings provide routing and other configuration data (such as sentinel,
+persistence policies, and other Redis customization) for connections
+to Redis single instances, Redis sentinel, and Redis clusters.
+
+If desired, the routing URL provided by these settings can be used with:
+1. Unix Socket
+ 1. named socket for each Redis instance desired.
+ 2. `database number` for each Redis instance desired.
+2. TCP Socket
+ 1. `host name` or IP for each Redis instance desired
+ 2. TCP port number for each Redis instance desired
+ 3. `database number` for each Redis instance desired
+
+## Example URL attribute formats for GitLab Redis `.yml` configuration files
+* Unix Socket, default Redis database (0)
+ * `url: unix:/path/to/redis.sock`
+ * `url: unix:/path/to/redis.sock?db=`
+* Unix Socket, Redis database 44
+ * `url: unix:/path/to/redis.sock?db=44`
+ * `url: unix:/path/to/redis.sock?extra=foo&db=44`
+* TCP Socket for Redis on localhost, port 6379, database 33
+ * `url: redis://:mynewpassword@localhost:6379/33`
+* TCP Socket for Redis on remote host `myserver`, port 6379, database 33
+ * `url: redis://:mynewpassword@myserver:6379/33`
+
+## redis.cache.yml
+
+If configured, `redis.cache.yml` overrides the
+`resque.yml` settings to configure the Redis database instance
+used for `Rails.cache` and other volatile non-persistent data which enhances
+the performance of GitLab.
+Settings here can be overridden by the environment variable
+`GITLAB_REDIS_CACHE_CONFIG_FILE` which provides
+an alternate location for configuration settings.
+
+The order of precedence for the URL used to connect to the Redis instance
+used for `cache` is:
+1. URL from a configuration file pointed to by the
+`GITLAB_REDIS_CACHE_CONFIG_FILE` environment variable
+2. URL from `redis.cache.yml`
+3. URL from a configuration file pointed to by the
+`GITLAB_REDIS_CONFIG_FILE` environment variable
+4. URL from `resque.yml`
+5. `redis://localhost:6380`
+
+The order of precedence for all other configuration settings for `cache`
+are selected from only the first of the following files found (if a setting
+is not provided in an earlier file, the remainder of the files are not
+searched):
+1. the configuration file pointed to by the
+`GITLAB_REDIS_CACHE_CONFIG_FILE` environment variable
+2. the configuration file `redis.cache.yml`
+3. the configuration file pointed to by the
+`GITLAB_REDIS_CONFIG_FILE` environment variable
+4. the configuration file `resque.yml`
+
+## redis.queues.yml
+
+If configured, `redis.queues.yml` overrides the
+`resque.yml` settings to configure the Redis database instance
+used for clients of `::Gitlab::Redis::Queues`.
+These queues are intended to be the foundation
+of reliable inter-process communication between modules, whether on the same
+host node, or within a cluster. The primary clients of the queues are
+SideKiq, Mailroom, CI Runner, Workhorse, and push services. Settings here can
+be overridden by the environment variable
+`GITLAB_REDIS_QUEUES_CONFIG_FILE` which provides an alternate location for
+configuration settings.
+
+The order of precedence for the URL used to connect to the Redis instance
+used for `queues` is:
+1. URL from a configuration file pointed to by the
+`GITLAB_REDIS_QUEUES_CONFIG_FILE` environment variable
+2. URL from `redis.queues.yml`
+3. URL from a configuration file pointed to by the
+`GITLAB_REDIS_CONFIG_FILE` environment variable
+4. URL from `resque.yml`
+5. `redis://localhost:6381`
+
+The order of precedence for all other configuration settings for `queues`
+are selected from only the first of the following files found (if a setting
+is not provided in an earlier file, the remainder of the files are not
+searched):
+1. the configuration file pointed to by the
+`GITLAB_REDIS_QUEUES_CONFIG_FILE` environment variable
+2. the configuration file `redis.queues.yml`
+3. the configuration file pointed to by the
+`GITLAB_REDIS_CONFIG_FILE` environment variable
+4. the configuration file `resque.yml`
+
+## redis.shared_state.yml
+
+If configured, `redis.shared_state.yml` overrides the
+`resque.yml` settings to configure the Redis database instance
+used for clients of `::Gitlab::Redis::SharedState` such as session state,
+and rate limiting.
+Settings here can be overridden by the environment variable
+`GITLAB_REDIS_SHARED_STATE_CONFIG_FILE` which provides
+an alternate location for configuration settings.
+
+The order of precedence for the URL used to connect to the Redis instance
+used for `shared_state` is:
+1. URL from a configuration file pointed to by the
+`GITLAB_REDIS_SHARED_STATE_CONFIG_FILE` environment variable
+2. URL from `redis.shared_state.yml`
+3. URL from a configuration file pointed to by the
+`GITLAB_REDIS_CONFIG_FILE` environment variable
+4. URL from `resque.yml`
+5. `redis://localhost:6382`
+
+The order of precedence for all other configuration settings for `shared_state`
+are selected from only the first of the following files found (if a setting
+is not provided in an earlier file, the remainder of the files are not
+searched):
+1. the configuration file pointed to by the
+`GITLAB_REDIS_SHARED_STATE_CONFIG_FILE` environment variable
+2. the configuration file `redis.shared_state.yml`
+3. the configuration file pointed to by the
+`GITLAB_REDIS_CONFIG_FILE` environment variable
+4. the configuration file `resque.yml`
+
diff --git a/config/application.rb b/config/application.rb
index 2f4e26241951ffc6cda954f675a1461c86af8b50..1c13cc81270647ca72ce9f17e6427a5317496316 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -6,7 +6,9 @@
module Gitlab
class Application < Rails::Application
- require_dependency Rails.root.join('lib/gitlab/redis')
+ require_dependency Rails.root.join('lib/gitlab/redis/cache')
+ require_dependency Rails.root.join('lib/gitlab/redis/queues')
+ require_dependency Rails.root.join('lib/gitlab/redis/shared_state')
require_dependency Rails.root.join('lib/gitlab/request_context')
# Settings in config/environments/* take precedence over those specified here.
@@ -142,15 +144,15 @@ class Application < Rails::Application
end
end
- # Use Redis caching across all environments
- redis_config_hash = Gitlab::Redis.params
- redis_config_hash[:namespace] = Gitlab::Redis::CACHE_NAMESPACE
- redis_config_hash[:expires_in] = 2.weeks # Cache should not grow forever
+ # Use caching across all environments
+ caching_config_hash = Gitlab::Redis::Cache.params
+ caching_config_hash[:namespace] = Gitlab::Redis::Cache::CACHE_NAMESPACE
+ caching_config_hash[:expires_in] = 2.weeks # Cache should not grow forever
if Sidekiq.server? # threaded context
- redis_config_hash[:pool_size] = Sidekiq.options[:concurrency] + 5
- redis_config_hash[:pool_timeout] = 1
+ caching_config_hash[:pool_size] = Sidekiq.options[:concurrency] + 5
+ caching_config_hash[:pool_timeout] = 1
end
- config.cache_store = :redis_store, redis_config_hash
+ config.cache_store = :redis_store, caching_config_hash
config.active_record.raise_in_transactional_callbacks = true
diff --git a/config/initializers/7_redis.rb b/config/initializers/7_redis.rb
index ae2ca258df188ac76c939623dde3b32c46f1e53e..af4967521b8091769bee594728898997a9f05b5a 100644
--- a/config/initializers/7_redis.rb
+++ b/config/initializers/7_redis.rb
@@ -1,3 +1,8 @@
-# Make sure we initialize a Redis connection pool before Sidekiq starts
-# multi-threaded execution.
-Gitlab::Redis.with { nil }
+# Make sure we initialize a Redis connection pool before multi-threaded
+# execution starts by
+# 1. Sidekiq
+# 2. Rails.cache
+# 3. HTTP clients
+Gitlab::Redis::Cache.with { nil }
+Gitlab::Redis::Queues.with { nil }
+Gitlab::Redis::SharedState.with { nil }
diff --git a/config/initializers/peek.rb b/config/initializers/peek.rb
index 65432caac2a66a2995b4131bfa13fb6b53857a67..da8282ec9243be7cd34a5c2f6cae8770d4ab4e24 100644
--- a/config/initializers/peek.rb
+++ b/config/initializers/peek.rb
@@ -1,4 +1,4 @@
-Rails.application.config.peek.adapter = :redis, { client: ::Redis.new(Gitlab::Redis.params) }
+Rails.application.config.peek.adapter = :redis, { client: ::Redis.new(Gitlab::Redis::Cache.params) }
Peek.into Peek::Views::Host
Peek.into Peek::Views::PerformanceBar
diff --git a/config/initializers/session_store.rb b/config/initializers/session_store.rb
index 8919f7640fe14850e3e5dd991ae6b93f75d9fe52..e8213ac8ba4ec74010b33e23aa3968079bdc7ae7 100644
--- a/config/initializers/session_store.rb
+++ b/config/initializers/session_store.rb
@@ -19,12 +19,12 @@
if Rails.env.test?
Gitlab::Application.config.session_store :cookie_store, key: "_gitlab_session"
else
- redis_config = Gitlab::Redis.params
- redis_config[:namespace] = Gitlab::Redis::SESSION_NAMESPACE
+ sessions_config = Gitlab::Redis::SharedState.params
+ sessions_config[:namespace] = Gitlab::Redis::SharedState::SESSION_NAMESPACE
Gitlab::Application.config.session_store(
:redis_store, # Using the cookie_store would enable session replay attacks.
- servers: redis_config,
+ servers: sessions_config,
key: cookie_key,
secure: Gitlab.config.gitlab.https,
httponly: true,
diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb
index 3be4cd797aa457014ff903e0ab996e34753bbc5e..a1cc9655319c5848e4234aa77a354c120161f1fb 100644
--- a/config/initializers/sidekiq.rb
+++ b/config/initializers/sidekiq.rb
@@ -1,12 +1,12 @@
-# Custom Redis configuration
-redis_config_hash = Gitlab::Redis.params
-redis_config_hash[:namespace] = Gitlab::Redis::SIDEKIQ_NAMESPACE
+# Custom Queues configuration
+queues_config_hash = Gitlab::Redis::Queues.params
+queues_config_hash[:namespace] = Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE
# Default is to retry 25 times with exponential backoff. That's too much.
Sidekiq.default_worker_options = { retry: 3 }
Sidekiq.configure_server do |config|
- config.redis = redis_config_hash
+ config.redis = queues_config_hash
config.server_middleware do |chain|
chain.add Gitlab::SidekiqMiddleware::ArgumentsLogger if ENV['SIDEKIQ_LOG_ARGUMENTS']
@@ -54,7 +54,7 @@
end
Sidekiq.configure_client do |config|
- config.redis = redis_config_hash
+ config.redis = queues_config_hash
config.client_middleware do |chain|
chain.add Gitlab::SidekiqStatus::ClientMiddleware
diff --git a/config/mail_room.yml b/config/mail_room.yml
index 88d93d4bc6b09495ca6ea5e353e30c9ab4ff54db..c3a5be8d38c5ab1a2f19b2500c42c9a1cc0535f5 100644
--- a/config/mail_room.yml
+++ b/config/mail_room.yml
@@ -21,7 +21,7 @@
:delivery_method: sidekiq
:delivery_options:
:redis_url: <%= config[:redis_url].to_json %>
- :namespace: <%= Gitlab::Redis::SIDEKIQ_NAMESPACE %>
+ :namespace: <%= Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE %>
:queue: email_receiver
:worker: EmailReceiverWorker
<% if config[:sentinels] %>
@@ -36,7 +36,7 @@
:arbitration_method: redis
:arbitration_options:
:redis_url: <%= config[:redis_url].to_json %>
- :namespace: <%= Gitlab::Redis::MAILROOM_NAMESPACE %>
+ :namespace: <%= Gitlab::Redis::Queues::MAILROOM_NAMESPACE %>
<% if config[:sentinels] %>
:sentinels:
<% config[:sentinels].each do |sentinel| %>
diff --git a/config/redis.cache.yml.example b/config/redis.cache.yml.example
new file mode 100644
index 0000000000000000000000000000000000000000..27478f0a93e48c85f3814d619f4891c81c95df77
--- /dev/null
+++ b/config/redis.cache.yml.example
@@ -0,0 +1,38 @@
+# If you change this file in a Merge Request, please also create
+# a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests
+#
+development:
+ url: redis://localhost:6379/10
+ #
+ # url: redis://localhost:6380
+ # sentinels:
+ # -
+ # host: localhost
+ # port: 26380 # point to sentinel, not to redis port
+ # -
+ # host: slave2
+ # port: 26380 # point to sentinel, not to redis port
+test:
+ url: redis://localhost:6379/10
+ #
+ # url: redis://localhost:6380
+production:
+ # Redis (single instance)
+ url: unix:/var/run/redis/redis.cache.sock
+ ##
+ # Redis + Sentinel (for HA)
+ #
+ # Please read instructions carefully before using it as you may lose data:
+ # http://redis.io/topics/sentinel
+ #
+ # You must specify a list of a few sentinels that will handle client connection
+ # please read here for more information: https://docs.gitlab.com/ce/administration/high_availability/redis.html
+ ##
+ # url: redis://master:6380
+ # sentinels:
+ # -
+ # host: slave1
+ # port: 26380 # point to sentinel, not to redis port
+ # -
+ # host: slave2
+ # port: 26380 # point to sentinel, not to redis port
diff --git a/config/redis.queues.yml.example b/config/redis.queues.yml.example
new file mode 100644
index 0000000000000000000000000000000000000000..dab1f26b0964b2cf91d798910826139c72dfebb3
--- /dev/null
+++ b/config/redis.queues.yml.example
@@ -0,0 +1,38 @@
+# If you change this file in a Merge Request, please also create
+# a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests
+#
+development:
+ url: redis://localhost:6379/11
+ #
+ # url: redis://localhost:6381
+ # sentinels:
+ # -
+ # host: localhost
+ # port: 26381 # point to sentinel, not to redis port
+ # -
+ # host: slave2
+ # port: 26381 # point to sentinel, not to redis port
+test:
+ url: redis://localhost:6379/11
+ #
+ # url: redis://localhost:6381
+production:
+ # Redis (single instance)
+ url: unix:/var/run/redis/redis.queues.sock
+ ##
+ # Redis + Sentinel (for HA)
+ #
+ # Please read instructions carefully before using it as you may lose data:
+ # http://redis.io/topics/sentinel
+ #
+ # You must specify a list of a few sentinels that will handle client connection
+ # please read here for more information: https://docs.gitlab.com/ce/administration/high_availability/redis.html
+ ##
+ # url: redis://master:6381
+ # sentinels:
+ # -
+ # host: slave1
+ # port: 26381 # point to sentinel, not to redis port
+ # -
+ # host: slave2
+ # port: 26381 # point to sentinel, not to redis port
diff --git a/config/redis.shared_state.yml.example b/config/redis.shared_state.yml.example
new file mode 100644
index 0000000000000000000000000000000000000000..9371e3619b787849358c525c0818eb7b4b8f78cf
--- /dev/null
+++ b/config/redis.shared_state.yml.example
@@ -0,0 +1,38 @@
+# If you change this file in a Merge Request, please also create
+# a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests
+#
+development:
+ url: redis://localhost:6379/12
+ #
+ # url: redis://localhost:6382
+ # sentinels:
+ # -
+ # host: localhost
+ # port: 26382 # point to sentinel, not to redis port
+ # -
+ # host: slave2
+ # port: 26382 # point to sentinel, not to redis port
+test:
+ url: redis://localhost:6379/12
+ #
+ # url: redis://localhost:6382
+production:
+ # Redis (single instance)
+ url: unix:/var/run/redis/redis.shared_state.sock
+ ##
+ # Redis + Sentinel (for HA)
+ #
+ # Please read instructions carefully before using it as you may lose data:
+ # http://redis.io/topics/sentinel
+ #
+ # You must specify a list of a few sentinels that will handle client connection
+ # please read here for more information: https://docs.gitlab.com/ce/administration/high_availability/redis.html
+ ##
+ # url: redis://master:6382
+ # sentinels:
+ # -
+ # host: slave1
+ # port: 26382 # point to sentinel, not to redis port
+ # -
+ # host: slave2
+ # port: 26382 # point to sentinel, not to redis port
diff --git a/db/post_migrate/20170324160416_migrate_user_activities_to_users_last_activity_on.rb b/db/post_migrate/20170324160416_migrate_user_activities_to_users_last_activity_on.rb
index 397a9a2d28ea6cec8469d0fa3565c0a8a9094ce2..cb1b4f1855da9c998f7a4cf633c0e08082e6d52d 100644
--- a/db/post_migrate/20170324160416_migrate_user_activities_to_users_last_activity_on.rb
+++ b/db/post_migrate/20170324160416_migrate_user_activities_to_users_last_activity_on.rb
@@ -56,7 +56,7 @@ def users_table
end
def activities(from, to, page: 1)
- Gitlab::Redis.with do |redis|
+ Gitlab::Redis::SharedState.with do |redis|
redis.zrangebyscore(USER_ACTIVITY_SET_KEY, from.to_i, to.to_i,
with_scores: true,
limit: limit(page))
@@ -64,7 +64,7 @@ def activities(from, to, page: 1)
end
def activities_count(from, to)
- Gitlab::Redis.with do |redis|
+ Gitlab::Redis::SharedState.with do |redis|
redis.zcount(USER_ACTIVITY_SET_KEY, from.to_i, to.to_i)
end
end
diff --git a/doc/administration/high_availability/redis_source.md b/doc/administration/high_availability/redis_source.md
index fe982ea83c2315545e0950a1378c163e0d059c9e..8b7a515a076771623c5edcf9f7741e3277347b9f 100644
--- a/doc/administration/high_availability/redis_source.md
+++ b/doc/administration/high_availability/redis_source.md
@@ -4,6 +4,11 @@ This is the documentation for configuring a Highly Available Redis setup when
you have installed Redis all by yourself and not using the bundled one that
comes with the Omnibus packages.
+Note also that you may elect to override all references to
+`/home/git/gitlab/config/resque.yml` in accordance with the advanced Redis
+settings outlined in
+[Configuration Files Documentation](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/README.md).
+
We cannot stress enough the importance of reading the
[Overview section](redis.md#overview) of the Omnibus Redis HA as it provides
some invaluable information to the configuration of Redis. Please proceed to
diff --git a/doc/administration/operations/cleaning_up_redis_sessions.md b/doc/administration/operations/cleaning_up_redis_sessions.md
index 93521e976d51e05204c34addbedc8cb0e4c514ab..3a35aff836676ca92e4284aca8c18347d6761a2b 100644
--- a/doc/administration/operations/cleaning_up_redis_sessions.md
+++ b/doc/administration/operations/cleaning_up_redis_sessions.md
@@ -15,6 +15,12 @@ prefixed with 'session:gitlab:', so they would look like
'session:gitlab:976aa289e2189b17d7ef525a6702ace9'. Below we describe how to
remove the keys in the old format.
+**Note:** the instructions below must be modified in accordance with your
+configuration settings if you have used the advanced Redis
+settings outlined in
+[Configuration Files Documentation](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/README.md).
+
+
First we define a shell function with the proper Redis connection details.
```
diff --git a/doc/install/installation.md b/doc/install/installation.md
index dfa25deb961c4cd5df52415e52fe548ef96dbe35..992ff162efb706e939d21c59af3e6c19c134d92c 100644
--- a/doc/install/installation.md
+++ b/doc/install/installation.md
@@ -420,12 +420,6 @@ GitLab Shell is an SSH access and repository management software developed speci
**Note:** Make sure your hostname can be resolved on the machine itself by either a proper DNS record or an additional line in /etc/hosts ("127.0.0.1 hostname"). This might be necessary for example if you set up GitLab behind a reverse proxy. If the hostname cannot be resolved, the final installation check will fail with "Check GitLab API access: FAILED. code: 401" and pushing commits will be rejected with "[remote rejected] master -> master (hook declined)".
-**Note:** GitLab Shell application startup time can be greatly reduced by disabling RubyGems. This can be done in several manners:
-
-* Export `RUBYOPT=--disable-gems` environment variable for the processes
-* Compile Ruby with `configure --disable-rubygems` to disable RubyGems by default. Not recommened for system-wide Ruby.
-* Omnibus GitLab [replaces the *shebang* line of the `gitlab-shell/bin/*` scripts](https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests/1707)
-
### Install gitlab-workhorse
GitLab-Workhorse uses [GNU Make](https://www.gnu.org/software/make/). The
diff --git a/lib/gitlab/auth/unique_ips_limiter.rb b/lib/gitlab/auth/unique_ips_limiter.rb
index bf2239ca150e0dbba7d8878240712a2a6b7fc4df..baa1f802d8a0c7989d63d922b2ca00b659ad589e 100644
--- a/lib/gitlab/auth/unique_ips_limiter.rb
+++ b/lib/gitlab/auth/unique_ips_limiter.rb
@@ -27,7 +27,7 @@ def update_and_return_ips_count(user_id, ip)
time = Time.now.utc.to_i
key = "#{USER_UNIQUE_IPS_PREFIX}:#{user_id}"
- Gitlab::Redis.with do |redis|
+ Gitlab::Redis::SharedState.with do |redis|
unique_ips_count = nil
redis.multi do |r|
r.zadd(key, time, ip)
diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb
index 9c2e09943b02b95d089829759097a03fa70bb9f5..dba378928639814b014a9aaa9d64419c28c26239 100644
--- a/lib/gitlab/cache/ci/project_pipeline_status.rb
+++ b/lib/gitlab/cache/ci/project_pipeline_status.rb
@@ -23,7 +23,7 @@ def self.load_in_batch_for_projects(projects)
end
def self.cached_results_for_projects(projects)
- result = Gitlab::Redis.with do |redis|
+ result = Gitlab::Redis::Cache.with do |redis|
redis.multi do
projects.each do |project|
cache_key = cache_key_for_project(project)
@@ -100,19 +100,19 @@ def store_in_cache_if_needed
end
def load_from_cache
- Gitlab::Redis.with do |redis|
+ Gitlab::Redis::Cache.with do |redis|
self.sha, self.status, self.ref = redis.hmget(cache_key, :sha, :status, :ref)
end
end
def store_in_cache
- Gitlab::Redis.with do |redis|
+ Gitlab::Redis::Cache.with do |redis|
redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref })
end
end
def delete_from_cache
- Gitlab::Redis.with do |redis|
+ Gitlab::Redis::Cache.with do |redis|
redis.del(cache_key)
end
end
@@ -120,7 +120,7 @@ def delete_from_cache
def has_cache?
return self.loaded unless self.loaded.nil?
- Gitlab::Redis.with do |redis|
+ Gitlab::Redis::Cache.with do |redis|
redis.exists(cache_key)
end
end
diff --git a/lib/gitlab/chat_name_token.rb b/lib/gitlab/chat_name_token.rb
index 1b081aa9b1d7d281c06438cb5522dd299a31e10f..e63e54373313ccabef85dfef5177b8853df51917 100644
--- a/lib/gitlab/chat_name_token.rb
+++ b/lib/gitlab/chat_name_token.rb
@@ -12,23 +12,23 @@ def initialize(token = new_token)
end
def get
- Gitlab::Redis.with do |redis|
- data = redis.get(redis_key)
+ Gitlab::Redis::SharedState.with do |redis|
+ data = redis.get(redis_shared_state_key)
JSON.parse(data, symbolize_names: true) if data
end
end
def store!(params)
- Gitlab::Redis.with do |redis|
+ Gitlab::Redis::SharedState.with do |redis|
params = params.to_json
- redis.set(redis_key, params, ex: EXPIRY_TIME)
+ redis.set(redis_shared_state_key, params, ex: EXPIRY_TIME)
token
end
end
def delete
- Gitlab::Redis.with do |redis|
- redis.del(redis_key)
+ Gitlab::Redis::SharedState.with do |redis|
+ redis.del(redis_shared_state_key)
end
end
@@ -38,7 +38,7 @@ def new_token
Devise.friendly_token(TOKEN_LENGTH)
end
- def redis_key
+ def redis_shared_state_key
"gitlab:chat_names:#{token}"
end
end
diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb
index 7fa02f3d7b38fcc95ee49a13c40ac30f4d3dddc5..791a3c36476f6f3d30f5e628493a92224c976891 100644
--- a/lib/gitlab/current_settings.rb
+++ b/lib/gitlab/current_settings.rb
@@ -25,7 +25,7 @@ def ensure_application_settings!
def cached_application_settings
begin
::ApplicationSetting.cached
- rescue ::Redis::BaseError, ::Errno::ENOENT, ::Errno::EADDRNOTAVAIL
+ rescue ::Redis::BaseError, ::Errno::ENOENT
# In case Redis isn't running or the Redis UNIX socket file is not available
end
end
diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb
index 33f8939bc61fc2abefdfd1cbd698d8206ca79599..1a697396ff1f832c7eace7c14660bc69508c1976 100644
--- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb
+++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb
@@ -145,7 +145,7 @@ def remove_cached_html_for_projects(project_ids)
def track_rename(type, old_path, new_path)
key = redis_key_for_type(type)
- Gitlab::Redis.with do |redis|
+ Gitlab::Redis::SharedState.with do |redis|
redis.lpush(key, [old_path, new_path].to_json)
redis.expire(key, 2.weeks.to_i)
end
@@ -155,7 +155,7 @@ def track_rename(type, old_path, new_path)
def reverts_for_type(type)
key = redis_key_for_type(type)
- Gitlab::Redis.with do |redis|
+ Gitlab::Redis::SharedState.with do |redis|
failed_reverts = []
while rename_info = redis.lpop(key)
diff --git a/lib/gitlab/etag_caching/store.rb b/lib/gitlab/etag_caching/store.rb
index 072fcfc65e600d6dd0ac541a3081b748f0ac8ae9..21172ff8d93cb50b419d6e35eb73090d4c073752 100644
--- a/lib/gitlab/etag_caching/store.rb
+++ b/lib/gitlab/etag_caching/store.rb
@@ -2,17 +2,17 @@ module Gitlab
module EtagCaching
class Store
EXPIRY_TIME = 20.minutes
- REDIS_NAMESPACE = 'etag:'.freeze
+ SHARED_STATE_NAMESPACE = 'etag:'.freeze
def get(key)
- Gitlab::Redis.with { |redis| redis.get(redis_key(key)) }
+ Gitlab::Redis::SharedState.with { |redis| redis.get(redis_shared_state_key(key)) }
end
def touch(key, only_if_missing: false)
etag = generate_etag
- Gitlab::Redis.with do |redis|
- redis.set(redis_key(key), etag, ex: EXPIRY_TIME, nx: only_if_missing)
+ Gitlab::Redis::SharedState.with do |redis|
+ redis.set(redis_shared_state_key(key), etag, ex: EXPIRY_TIME, nx: only_if_missing)
end
etag
@@ -24,10 +24,10 @@ def generate_etag
SecureRandom.hex
end
- def redis_key(key)
+ def redis_shared_state_key(key)
raise 'Invalid key' if !Rails.env.production? && !Gitlab::EtagCaching::Router.match(key)
- "#{REDIS_NAMESPACE}#{key}"
+ "#{SHARED_STATE_NAMESPACE}#{key}"
end
end
end
diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb
index a0f46594eb1cb460a6f627c474c12fd784577eb5..3784f6c4947ed7683a9507eca990e568eea6a3c5 100644
--- a/lib/gitlab/exclusive_lease.rb
+++ b/lib/gitlab/exclusive_lease.rb
@@ -26,17 +26,17 @@ class ExclusiveLease
EOS
def self.cancel(key, uuid)
- Gitlab::Redis.with do |redis|
- redis.eval(LUA_CANCEL_SCRIPT, keys: [redis_key(key)], argv: [uuid])
+ Gitlab::Redis::SharedState.with do |redis|
+ redis.eval(LUA_CANCEL_SCRIPT, keys: [redis_shared_state_key(key)], argv: [uuid])
end
end
- def self.redis_key(key)
+ def self.redis_shared_state_key(key)
"gitlab:exclusive_lease:#{key}"
end
def initialize(key, timeout:)
- @redis_key = self.class.redis_key(key)
+ @redis_shared_state_key = self.class.redis_shared_state_key(key)
@timeout = timeout
@uuid = SecureRandom.uuid
end
@@ -45,24 +45,24 @@ def initialize(key, timeout:)
# false if the lease is already taken.
def try_obtain
# Performing a single SET is atomic
- Gitlab::Redis.with do |redis|
- redis.set(@redis_key, @uuid, nx: true, ex: @timeout) && @uuid
+ Gitlab::Redis::SharedState.with do |redis|
+ redis.set(@redis_shared_state_key, @uuid, nx: true, ex: @timeout) && @uuid
end
end
# Try to renew an existing lease. Return lease UUID on success,
# false if the lease is taken by a different UUID or inexistent.
def renew
- Gitlab::Redis.with do |redis|
- result = redis.eval(LUA_RENEW_SCRIPT, keys: [@redis_key], argv: [@uuid, @timeout])
+ Gitlab::Redis::SharedState.with do |redis|
+ result = redis.eval(LUA_RENEW_SCRIPT, keys: [@redis_shared_state_key], argv: [@uuid, @timeout])
result == @uuid
end
end
# Returns true if the key for this lease is set.
def exists?
- Gitlab::Redis.with do |redis|
- redis.exists(@redis_key)
+ Gitlab::Redis::SharedState.with do |redis|
+ redis.exists(@redis_shared_state_key)
end
end
end
diff --git a/lib/gitlab/health_checks/redis/cache_check.rb b/lib/gitlab/health_checks/redis/cache_check.rb
new file mode 100644
index 0000000000000000000000000000000000000000..a28658d42d439be3d7d40fef81a0f90b2e183906
--- /dev/null
+++ b/lib/gitlab/health_checks/redis/cache_check.rb
@@ -0,0 +1,31 @@
+module Gitlab
+ module HealthChecks
+ module Redis
+ class CacheCheck
+ extend SimpleAbstractCheck
+
+ class << self
+ def check_up
+ check
+ end
+
+ private
+
+ def metric_prefix
+ 'redis_cache_ping'
+ end
+
+ def is_successful?(result)
+ result == 'PONG'
+ end
+
+ def check
+ catch_timeout 10.seconds do
+ Gitlab::Redis::Cache.with(&:ping)
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/health_checks/redis/queues_check.rb b/lib/gitlab/health_checks/redis/queues_check.rb
new file mode 100644
index 0000000000000000000000000000000000000000..f97d50d394794e170ba1a29dde18de26bd840b42
--- /dev/null
+++ b/lib/gitlab/health_checks/redis/queues_check.rb
@@ -0,0 +1,31 @@
+module Gitlab
+ module HealthChecks
+ module Redis
+ class QueuesCheck
+ extend SimpleAbstractCheck
+
+ class << self
+ def check_up
+ check
+ end
+
+ private
+
+ def metric_prefix
+ 'redis_queues_ping'
+ end
+
+ def is_successful?(result)
+ result == 'PONG'
+ end
+
+ def check
+ catch_timeout 10.seconds do
+ Gitlab::Redis::Queues.with(&:ping)
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/health_checks/redis/redis_check.rb b/lib/gitlab/health_checks/redis/redis_check.rb
new file mode 100644
index 0000000000000000000000000000000000000000..fe4e3c4a3ab51bb414e31b370a23b8bc66c911bd
--- /dev/null
+++ b/lib/gitlab/health_checks/redis/redis_check.rb
@@ -0,0 +1,27 @@
+module Gitlab
+ module HealthChecks
+ module Redis
+ class RedisCheck
+ extend SimpleAbstractCheck
+
+ class << self
+ private
+
+ def metric_prefix
+ 'redis_ping'
+ end
+
+ def is_successful?(result)
+ result == 'PONG'
+ end
+
+ def check
+ ::Gitlab::HealthChecks::Redis::CacheCheck.check_up &&
+ ::Gitlab::HealthChecks::Redis::QueuesCheck.check_up &&
+ ::Gitlab::HealthChecks::Redis::SharedStateCheck.check_up
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/health_checks/redis/shared_state_check.rb b/lib/gitlab/health_checks/redis/shared_state_check.rb
new file mode 100644
index 0000000000000000000000000000000000000000..e3244392902a67af46977972ae69a2e9141e5797
--- /dev/null
+++ b/lib/gitlab/health_checks/redis/shared_state_check.rb
@@ -0,0 +1,31 @@
+module Gitlab
+ module HealthChecks
+ module Redis
+ class SharedStateCheck
+ extend SimpleAbstractCheck
+
+ class << self
+ def check_up
+ check
+ end
+
+ private
+
+ def metric_prefix
+ 'redis_shared_state_ping'
+ end
+
+ def is_successful?(result)
+ result == 'PONG'
+ end
+
+ def check
+ catch_timeout 10.seconds do
+ Gitlab::Redis::SharedState.with(&:ping)
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/health_checks/redis_check.rb b/lib/gitlab/health_checks/redis_check.rb
deleted file mode 100644
index 57bbe5b3ad03fe2fd517931c0a4981b0953db4c0..0000000000000000000000000000000000000000
--- a/lib/gitlab/health_checks/redis_check.rb
+++ /dev/null
@@ -1,25 +0,0 @@
-module Gitlab
- module HealthChecks
- class RedisCheck
- extend SimpleAbstractCheck
-
- class << self
- private
-
- def metric_prefix
- 'redis_ping'
- end
-
- def is_successful?(result)
- result == 'PONG'
- end
-
- def check
- catch_timeout 10.seconds do
- Gitlab::Redis.with(&:ping)
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb
index 5f67e97fa2a718ad0ff8e004c281834d2b7d94f7..8e57ba831c51591363061804fabe5e30bf9fac7f 100644
--- a/lib/gitlab/lfs_token.rb
+++ b/lib/gitlab/lfs_token.rb
@@ -18,10 +18,10 @@ def initialize(actor)
end
def token
- Gitlab::Redis.with do |redis|
- token = redis.get(redis_key)
+ Gitlab::Redis::SharedState.with do |redis|
+ token = redis.get(redis_shared_state_key)
token ||= Devise.friendly_token(TOKEN_LENGTH)
- redis.set(redis_key, token, ex: EXPIRY_TIME)
+ redis.set(redis_shared_state_key, token, ex: EXPIRY_TIME)
token
end
@@ -41,7 +41,7 @@ def actor_name
private
- def redis_key
+ def redis_shared_state_key
"gitlab:lfs_token:#{actor.class.name.underscore}_#{actor.id}" if actor
end
end
diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb
index 3503fac40e8bfe3788d850bcbe994a719b672472..9f432673a6ecd58d01d18301464885700cc14e62 100644
--- a/lib/gitlab/mail_room.rb
+++ b/lib/gitlab/mail_room.rb
@@ -1,6 +1,6 @@
require 'yaml'
require 'json'
-require_relative 'redis' unless defined?(Gitlab::Redis)
+require_relative 'redis/queues' unless defined?(Gitlab::Redis::Queues)
module Gitlab
module MailRoom
@@ -34,11 +34,11 @@ def fetch_config
config[:idle_timeout] = 60 if config[:idle_timeout].nil?
if config[:enabled] && config[:address]
- gitlab_redis = Gitlab::Redis.new(rails_env)
- config[:redis_url] = gitlab_redis.url
+ gitlab_redis_queues = Gitlab::Redis::Queues.new(rails_env)
+ config[:redis_url] = gitlab_redis_queues.url
- if gitlab_redis.sentinels?
- config[:sentinels] = gitlab_redis.sentinels
+ if gitlab_redis_queues.sentinels?
+ config[:sentinels] = gitlab_redis_queues.sentinels
end
end
diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb
deleted file mode 100644
index bc5370de32aa5fd8f1ed126c99a2be1b40ff7855..0000000000000000000000000000000000000000
--- a/lib/gitlab/redis.rb
+++ /dev/null
@@ -1,102 +0,0 @@
-# This file should not have any direct dependency on Rails environment
-# please require all dependencies below:
-require 'active_support/core_ext/hash/keys'
-require 'active_support/core_ext/module/delegation'
-
-module Gitlab
- class Redis
- CACHE_NAMESPACE = 'cache:gitlab'.freeze
- SESSION_NAMESPACE = 'session:gitlab'.freeze
- SIDEKIQ_NAMESPACE = 'resque:gitlab'.freeze
- MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze
- DEFAULT_REDIS_URL = 'redis://localhost:6379'.freeze
-
- class << self
- delegate :params, :url, to: :new
-
- def with
- @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) }
- @pool.with { |redis| yield redis }
- end
-
- def pool_size
- if Sidekiq.server?
- # the pool will be used in a multi-threaded context
- Sidekiq.options[:concurrency] + 5
- else
- # probably this is a Unicorn process, so single threaded
- 5
- end
- end
-
- def _raw_config
- return @_raw_config if defined?(@_raw_config)
-
- begin
- @_raw_config = ERB.new(File.read(config_file)).result.freeze
- rescue Errno::ENOENT
- @_raw_config = false
- end
-
- @_raw_config
- end
-
- def config_file
- ENV['GITLAB_REDIS_CONFIG_FILE'] || File.expand_path('../../config/resque.yml', __dir__)
- end
- end
-
- def initialize(rails_env = nil)
- @rails_env = rails_env || ::Rails.env
- end
-
- def params
- redis_store_options
- end
-
- def url
- raw_config_hash[:url]
- end
-
- def sentinels
- raw_config_hash[:sentinels]
- end
-
- def sentinels?
- sentinels && !sentinels.empty?
- end
-
- private
-
- def redis_store_options
- config = raw_config_hash
- redis_url = config.delete(:url)
- redis_uri = URI.parse(redis_url)
-
- if redis_uri.scheme == 'unix'
- # Redis::Store does not handle Unix sockets well, so let's do it for them
- config[:path] = redis_uri.path
- config
- else
- redis_hash = ::Redis::Store::Factory.extract_host_options_from_uri(redis_url)
- # order is important here, sentinels must be after the connection keys.
- # {url: ..., port: ..., sentinels: [...]}
- redis_hash.merge(config)
- end
- end
-
- def raw_config_hash
- config_data = fetch_config
-
- if config_data
- config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys
- else
- { url: DEFAULT_REDIS_URL }
- end
- end
-
- def fetch_config
- self.class._raw_config ? YAML.load(self.class._raw_config)[@rails_env] : false
- end
- end
-end
diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb
new file mode 100644
index 0000000000000000000000000000000000000000..b0da516ff83e14f04a5d2695d1defe5c9cfb0219
--- /dev/null
+++ b/lib/gitlab/redis/cache.rb
@@ -0,0 +1,34 @@
+# please require all dependencies below:
+require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper)
+
+module Gitlab
+ module Redis
+ class Cache < ::Gitlab::Redis::Wrapper
+ CACHE_NAMESPACE = 'cache:gitlab'.freeze
+ DEFAULT_REDIS_CACHE_URL = 'redis://localhost:6380'.freeze
+ REDIS_CACHE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CACHE_CONFIG_FILE'.freeze
+ if defined?(::Rails) && ::Rails.root.present?
+ DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.cache.yml').freeze
+ end
+
+ class << self
+ def default_url
+ DEFAULT_REDIS_CACHE_URL
+ end
+
+ def config_file_name
+ # if ENV set for this class, use it even if it points to a file does not exist
+ file_name = ENV[REDIS_CACHE_CONFIG_ENV_VAR_NAME]
+ return file_name unless file_name.nil?
+
+ # otherwise, if config files exists for this class, use it
+ file_name = File.expand_path(DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME, __dir__)
+ return file_name if File.file?(file_name)
+
+ # this will force use of DEFAULT_REDIS_QUEUES_URL when config file is absent
+ super
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/redis/queues.rb b/lib/gitlab/redis/queues.rb
new file mode 100644
index 0000000000000000000000000000000000000000..f9249d055653933bce3d16e0903c4e5c7836e2e3
--- /dev/null
+++ b/lib/gitlab/redis/queues.rb
@@ -0,0 +1,35 @@
+# please require all dependencies below:
+require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper)
+
+module Gitlab
+ module Redis
+ class Queues < ::Gitlab::Redis::Wrapper
+ SIDEKIQ_NAMESPACE = 'resque:gitlab'.freeze
+ MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze
+ DEFAULT_REDIS_QUEUES_URL = 'redis://localhost:6381'.freeze
+ REDIS_QUEUES_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_QUEUES_CONFIG_FILE'.freeze
+ if defined?(::Rails) && ::Rails.root.present?
+ DEFAULT_REDIS_QUEUES_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.queues.yml').freeze
+ end
+
+ class << self
+ def default_url
+ DEFAULT_REDIS_QUEUES_URL
+ end
+
+ def config_file_name
+ # if ENV set for this class, use it even if it points to a file does not exist
+ file_name = ENV[REDIS_QUEUES_CONFIG_ENV_VAR_NAME]
+ return file_name if file_name
+
+ # otherwise, if config files exists for this class, use it
+ file_name = File.expand_path(DEFAULT_REDIS_QUEUES_CONFIG_FILE_NAME, __dir__)
+ return file_name if File.file?(file_name)
+
+ # this will force use of DEFAULT_REDIS_QUEUES_URL when config file is absent
+ super
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb
new file mode 100644
index 0000000000000000000000000000000000000000..395dcf082da2a63a6eced5a75b9c094da6f3e3ff
--- /dev/null
+++ b/lib/gitlab/redis/shared_state.rb
@@ -0,0 +1,34 @@
+# please require all dependencies below:
+require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper)
+
+module Gitlab
+ module Redis
+ class SharedState < ::Gitlab::Redis::Wrapper
+ SESSION_NAMESPACE = 'session:gitlab'.freeze
+ DEFAULT_REDIS_SHARED_STATE_URL = 'redis://localhost:6382'.freeze
+ REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'.freeze
+ if defined?(::Rails) && ::Rails.root.present?
+ DEFAULT_REDIS_SHARED_STATE_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.shared_state.yml').freeze
+ end
+
+ class << self
+ def default_url
+ DEFAULT_REDIS_SHARED_STATE_URL
+ end
+
+ def config_file_name
+ # if ENV set for this class, use it even if it points to a file does not exist
+ file_name = ENV[REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME]
+ return file_name if file_name
+
+ # otherwise, if config files exists for this class, use it
+ file_name = File.expand_path(DEFAULT_REDIS_SHARED_STATE_CONFIG_FILE_NAME, __dir__)
+ return file_name if File.file?(file_name)
+
+ # this will force use of DEFAULT_REDIS_SHARED_STATE_URL when config file is absent
+ super
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb
new file mode 100644
index 0000000000000000000000000000000000000000..10c16a962ca1778f7898883bdb4c20ec419c5dcc
--- /dev/null
+++ b/lib/gitlab/redis/wrapper.rb
@@ -0,0 +1,123 @@
+# This file should only be used by sub-classes, not directly by any clients of the sub-classes
+# please require all dependencies below:
+require 'active_support/core_ext/hash/keys'
+require 'active_support/core_ext/module/delegation'
+
+module Gitlab
+ module Redis
+ class Wrapper
+ DEFAULT_REDIS_URL = 'redis://localhost:6379'.freeze
+ REDIS_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CONFIG_FILE'.freeze
+ if defined?(::Rails) && ::Rails.root.present?
+ DEFAULT_REDIS_CONFIG_FILE_NAME = ::Rails.root.join('config', 'resque.yml').freeze
+ end
+
+ class << self
+ delegate :params, :url, to: :new
+
+ def with
+ @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) }
+ @pool.with { |redis| yield redis }
+ end
+
+ def pool_size
+ # heuristic constant 5 should be a config setting somewhere -- related to CPU count?
+ size = 5
+ if Sidekiq.server?
+ # the pool will be used in a multi-threaded context
+ size += Sidekiq.options[:concurrency]
+ end
+ size
+ end
+
+ def _raw_config
+ return @_raw_config if defined?(@_raw_config)
+
+ begin
+ @_raw_config = ERB.new(File.read(config_file_name)).result.freeze
+ rescue Errno::ENOENT
+ @_raw_config = false
+ end
+
+ @_raw_config
+ end
+
+ def default_url
+ DEFAULT_REDIS_URL
+ end
+
+ def config_file_name
+ # if ENV set for wrapper class, use it even if it points to a file does not exist
+ file_name = ENV[REDIS_CONFIG_ENV_VAR_NAME]
+ return file_name unless file_name.nil?
+
+ # otherwise, if config files exists for wrapper class, use it
+ file_name = File.expand_path(DEFAULT_REDIS_CONFIG_FILE_NAME, __dir__)
+ return file_name if File.file?(file_name)
+
+ # nil will force use of DEFAULT_REDIS_URL when config file is absent
+ nil
+ end
+ end
+
+ def initialize(rails_env = nil)
+ @rails_env = rails_env || ::Rails.env
+ end
+
+ def params
+ redis_store_options
+ end
+
+ def url
+ raw_config_hash[:url]
+ end
+
+ def sentinels
+ raw_config_hash[:sentinels]
+ end
+
+ def sentinels?
+ sentinels && !sentinels.empty?
+ end
+
+ private
+
+ def redis_store_options
+ config = raw_config_hash
+ redis_url = config.delete(:url)
+ redis_uri = URI.parse(redis_url)
+
+ if redis_uri.scheme == 'unix'
+ # Redis::Store does not handle Unix sockets well, so let's do it for them
+ config[:path] = redis_uri.path
+ query = redis_uri.query
+ unless query.nil?
+ queries = CGI.parse(redis_uri.query)
+ db_numbers = queries["db"] if queries.key?("db")
+ config[:db] = db_numbers[0].to_i if db_numbers.any?
+ end
+ config
+ else
+ redis_hash = ::Redis::Store::Factory.extract_host_options_from_uri(redis_url)
+ # order is important here, sentinels must be after the connection keys.
+ # {url: ..., port: ..., sentinels: [...]}
+ redis_hash.merge(config)
+ end
+ end
+
+ def raw_config_hash
+ config_data = fetch_config
+
+ if config_data
+ config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys
+ else
+ { url: self.class.default_url }
+ end
+ end
+
+ def fetch_config
+ self.class._raw_config ? YAML.load(self.class._raw_config)[@rails_env] : false
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/user_activities.rb b/lib/gitlab/user_activities.rb
index eb36ab9fdedfc65d3a1c22fa0d56e1430d6025da..125488536e19408c280bf2f3db29d30ab93cdffb 100644
--- a/lib/gitlab/user_activities.rb
+++ b/lib/gitlab/user_activities.rb
@@ -6,13 +6,13 @@ class UserActivities
BATCH_SIZE = 500
def self.record(key, time = Time.now)
- Gitlab::Redis.with do |redis|
+ Gitlab::Redis::SharedState.with do |redis|
redis.hset(KEY, key, time.to_i)
end
end
def delete(*keys)
- Gitlab::Redis.with do |redis|
+ Gitlab::Redis::SharedState.with do |redis|
redis.hdel(KEY, keys)
end
end
@@ -21,7 +21,7 @@ def each
cursor = 0
loop do
cursor, pairs =
- Gitlab::Redis.with do |redis|
+ Gitlab::Redis::SharedState.with do |redis|
redis.hscan(KEY, cursor, count: BATCH_SIZE)
end
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index 4aef23b6aee1596c86c7b20a373cf213cfe5518c..ee4f2b773eabe91049ce207359186d25967f7293 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -176,7 +176,7 @@ def secret_path
end
def set_key_and_notify(key, value, expire: nil, overwrite: true)
- Gitlab::Redis.with do |redis|
+ Gitlab::Redis::Queues.with do |redis|
result = redis.set(key, value, ex: expire, nx: !overwrite)
if result
redis.publish(NOTIFICATION_CHANNEL, "#{key}=#{value}")
diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake
index 125a3d560d662ef078dfb1f3973c27500d2bd44f..564aa1419521fb57069b506b4f70c6a2bebb917b 100644
--- a/lib/tasks/cache.rake
+++ b/lib/tasks/cache.rake
@@ -5,12 +5,12 @@ namespace :cache do
desc "GitLab | Clear redis cache"
task redis: :environment do
- Gitlab::Redis.with do |redis|
+ Gitlab::Redis::Cache.with do |redis|
cursor = REDIS_SCAN_START_STOP
loop do
cursor, keys = redis.scan(
cursor,
- match: "#{Gitlab::Redis::CACHE_NAMESPACE}*",
+ match: "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}*",
count: REDIS_CLEAR_BATCH_SIZE
)
diff --git a/scripts/prepare_build.sh b/scripts/prepare_build.sh
index 03de59f27adcefd02f18497d153963f8bdcddea5..68114d149c4c1303353cef600ced01eb80c0b57a 100644
--- a/scripts/prepare_build.sh
+++ b/scripts/prepare_build.sh
@@ -12,9 +12,6 @@ fi
# gems could not be found under some circumstance. No idea why, hours wasted.
retry gem install knapsack fog-aws mime-types
-cp config/resque.yml.example config/resque.yml
-sed -i 's/localhost/redis/g' config/resque.yml
-
cp config/gitlab.yml.example config/gitlab.yml
# Determine the database by looking at the job name.
@@ -37,6 +34,18 @@ else # Assume it's mysql
sed -i 's/# host:.*/host: mysql/g' config/database.yml
fi
+cp config/resque.yml.example config/resque.yml
+sed -i 's/localhost/redis/g' config/resque.yml
+
+cp config/redis.cache.yml.example config/redis.cache.yml
+sed -i 's/localhost/redis/g' config/redis.cache.yml
+
+cp config/redis.queues.yml.example config/redis.queues.yml
+sed -i 's/localhost/redis/g' config/redis.queues.yml
+
+cp config/redis.shared_state.yml.example config/redis.shared_state.yml
+sed -i 's/localhost/redis/g' config/redis.shared_state.yml
+
if [ "$SETUP_DB" != "false" ]; then
bundle exec rake db:drop db:create db:schema:load db:migrate
diff --git a/spec/config/mail_room_spec.rb b/spec/config/mail_room_spec.rb
index 092048a62599afa31298ac3663f5a4b11fc1725a..a31e44fa9285703cf50f3ed1adc3ceb805e9ca7a 100644
--- a/spec/config/mail_room_spec.rb
+++ b/spec/config/mail_room_spec.rb
@@ -5,12 +5,12 @@
let(:mailroom_config_path) { 'config/mail_room.yml' }
let(:gitlab_config_path) { 'config/mail_room.yml' }
- let(:redis_config_path) { 'config/resque.yml' }
+ let(:queues_config_path) { 'config/redis.queues.yml' }
let(:configuration) do
vars = {
'MAIL_ROOM_GITLAB_CONFIG_FILE' => absolute_path(gitlab_config_path),
- 'GITLAB_REDIS_CONFIG_FILE' => absolute_path(redis_config_path)
+ 'GITLAB_REDIS_QUEUES_CONFIG_FILE' => absolute_path(queues_config_path)
}
cmd = "puts ERB.new(File.read(#{absolute_path(mailroom_config_path).inspect})).result"
@@ -21,12 +21,12 @@
end
before(:each) do
- stub_env('GITLAB_REDIS_CONFIG_FILE', absolute_path(redis_config_path))
- clear_redis_raw_config
+ stub_env('GITLAB_REDIS_QUEUES_CONFIG_FILE', absolute_path(queues_config_path))
+ clear_queues_raw_config
end
after(:each) do
- clear_redis_raw_config
+ clear_queues_raw_config
end
context 'when incoming email is disabled' do
@@ -39,9 +39,9 @@
context 'when incoming email is enabled' do
let(:gitlab_config_path) { 'spec/fixtures/config/mail_room_enabled.yml' }
- let(:redis_config_path) { 'spec/fixtures/config/redis_new_format_host.yml' }
+ let(:queues_config_path) { 'spec/fixtures/config/redis_queues_new_format_host.yml' }
- let(:gitlab_redis) { Gitlab::Redis.new(Rails.env) }
+ let(:gitlab_redis_queues) { Gitlab::Redis::Queues.new(Rails.env) }
it 'contains the intended configuration' do
expect(configuration[:mailboxes].length).to eq(1)
@@ -56,8 +56,8 @@
expect(mailbox[:name]).to eq('inbox')
expect(mailbox[:idle_timeout]).to eq(60)
- redis_url = gitlab_redis.url
- sentinels = gitlab_redis.sentinels
+ redis_url = gitlab_redis_queues.url
+ sentinels = gitlab_redis_queues.sentinels
expect(mailbox[:delivery_options][:redis_url]).to be_present
expect(mailbox[:delivery_options][:redis_url]).to eq(redis_url)
@@ -73,8 +73,8 @@
end
end
- def clear_redis_raw_config
- Gitlab::Redis.remove_instance_variable(:@_raw_config)
+ def clear_queues_raw_config
+ Gitlab::Redis::Queues.remove_instance_variable(:@_raw_config)
rescue NameError
# raised if @_raw_config was not set; ignore
end
diff --git a/spec/controllers/health_controller_spec.rb b/spec/controllers/health_controller_spec.rb
index e7c19b47a6ac09f256e6f5e7611d63220e975f15..241c36620d13cefe3d4d20e92d16ea1ba67b0ab2 100644
--- a/spec/controllers/health_controller_spec.rb
+++ b/spec/controllers/health_controller_spec.rb
@@ -19,7 +19,9 @@
it 'returns proper response' do
get :readiness
expect(json_response['db_check']['status']).to eq('ok')
- expect(json_response['redis_check']['status']).to eq('ok')
+ expect(json_response['cache_check']['status']).to eq('ok')
+ expect(json_response['queues_check']['status']).to eq('ok')
+ expect(json_response['shared_state_check']['status']).to eq('ok')
expect(json_response['fs_shards_check']['status']).to eq('ok')
expect(json_response['fs_shards_check']['labels']['shard']).to eq('default')
end
@@ -42,7 +44,9 @@
it 'returns proper response' do
get :liveness
expect(json_response['db_check']['status']).to eq('ok')
- expect(json_response['redis_check']['status']).to eq('ok')
+ expect(json_response['cache_check']['status']).to eq('ok')
+ expect(json_response['queues_check']['status']).to eq('ok')
+ expect(json_response['shared_state_check']['status']).to eq('ok')
expect(json_response['fs_shards_check']['status']).to eq('ok')
end
end
diff --git a/spec/controllers/metrics_controller_spec.rb b/spec/controllers/metrics_controller_spec.rb
index 044c9f179ed3f47eb2eb3b2db9e061e638bf7e09..c46dd92b454d339a5bdbe0f06c79cbce0a61e071 100644
--- a/spec/controllers/metrics_controller_spec.rb
+++ b/spec/controllers/metrics_controller_spec.rb
@@ -35,6 +35,30 @@
expect(response.body).to match(/^redis_ping_latency [0-9\.]+$/)
end
+ it 'returns Caching ping metrics' do
+ get :index
+
+ expect(response.body).to match(/^redis_cache_ping_timeout 0$/)
+ expect(response.body).to match(/^redis_cache_ping_success 1$/)
+ expect(response.body).to match(/^redis_cache_ping_latency [0-9\.]+$/)
+ end
+
+ it 'returns Queues ping metrics' do
+ get :index
+
+ expect(response.body).to match(/^redis_queues_ping_timeout 0$/)
+ expect(response.body).to match(/^redis_queues_ping_success 1$/)
+ expect(response.body).to match(/^redis_queues_ping_latency [0-9\.]+$/)
+ end
+
+ it 'returns SharedState ping metrics' do
+ get :index
+
+ expect(response.body).to match(/^redis_shared_state_ping_timeout 0$/)
+ expect(response.body).to match(/^redis_shared_state_ping_success 1$/)
+ expect(response.body).to match(/^redis_shared_state_ping_latency [0-9\.]+$/)
+ end
+
it 'returns file system check metrics' do
get :index
diff --git a/spec/controllers/sessions_controller_spec.rb b/spec/controllers/sessions_controller_spec.rb
index bf922260b2f7a61d2a82767b00a5851f698be114..2b4e8723b48c3c12618e015268a0415525bbabb3 100644
--- a/spec/controllers/sessions_controller_spec.rb
+++ b/spec/controllers/sessions_controller_spec.rb
@@ -47,7 +47,7 @@
end
end
- context 'when using valid password', :redis do
+ context 'when using valid password', :clean_gitlab_redis_shared_state do
include UserActivitiesHelpers
let(:user) { create(:user) }
diff --git a/spec/features/dashboard/issuables_counter_spec.rb b/spec/features/dashboard/issuables_counter_spec.rb
index 285724f4b48a5bda2b0eb0a9c83860eb00803d6d..6b666934563604429783fe297f5028f34dedface 100644
--- a/spec/features/dashboard/issuables_counter_spec.rb
+++ b/spec/features/dashboard/issuables_counter_spec.rb
@@ -1,6 +1,6 @@
require 'spec_helper'
-describe 'Navigation bar counter', feature: true, caching: true do
+describe 'Navigation bar counter', :use_clean_rails_memory_store_caching, feature: true do
let(:user) { create(:user) }
let(:project) { create(:empty_project, namespace: user.namespace) }
let(:issue) { create(:issue, project: project) }
diff --git a/spec/features/dashboard/projects_spec.rb b/spec/features/dashboard/projects_spec.rb
index 7ca002fc82181e6384efffd21cd8d72fb34fd990..bdba22fe9a97327a8237c412a0cb98126394bd95 100644
--- a/spec/features/dashboard/projects_spec.rb
+++ b/spec/features/dashboard/projects_spec.rb
@@ -61,7 +61,7 @@
end
end
- describe 'with a pipeline', redis: true do
+ describe "with a pipeline", clean_gitlab_redis_shared_state: true do
let(:pipeline) { create(:ci_pipeline, project: project, sha: project.commit.sha) }
before do
diff --git a/spec/features/groups/members/sort_members_spec.rb b/spec/features/groups/members/sort_members_spec.rb
index 169827f5d0d20e0e20959c874e7feb5e4879fbf2..92ff45e0cdcac1a856441e63f3629b1be2ca2f6c 100644
--- a/spec/features/groups/members/sort_members_spec.rb
+++ b/spec/features/groups/members/sort_members_spec.rb
@@ -68,7 +68,7 @@
expect(page).to have_css('.member-sort-dropdown .dropdown-toggle-text', text: 'Name, descending')
end
- scenario 'sorts by recent sign in', :redis do
+ scenario 'sorts by recent sign in', :clean_gitlab_redis_shared_state do
visit_members_list(sort: :recent_sign_in)
expect(first_member).to include(owner.name)
@@ -76,7 +76,7 @@
expect(page).to have_css('.member-sort-dropdown .dropdown-toggle-text', text: 'Recent sign in')
end
- scenario 'sorts by oldest sign in', :redis do
+ scenario 'sorts by oldest sign in', :clean_gitlab_redis_shared_state do
visit_members_list(sort: :oldest_sign_in)
expect(first_member).to include(developer.name)
diff --git a/spec/features/login_spec.rb b/spec/features/login_spec.rb
index a8055b21cee158338ca6ee0b88543981e302dc7e..2a2213b67ed1bf827d11628a4ec912116816c3d3 100644
--- a/spec/features/login_spec.rb
+++ b/spec/features/login_spec.rb
@@ -41,7 +41,7 @@
expect(page).to have_content('Your account has been blocked.')
end
- it 'does not update Devise trackable attributes', :redis do
+ it 'does not update Devise trackable attributes', :clean_gitlab_redis_shared_state do
user = create(:user, :blocked)
expect { gitlab_sign_in(user) }.not_to change { user.reload.sign_in_count }
@@ -55,7 +55,7 @@
expect(page).to have_content('Invalid Login or password.')
end
- it 'does not update Devise trackable attributes', :redis do
+ it 'does not update Devise trackable attributes', :clean_gitlab_redis_shared_state do
expect { gitlab_sign_in(User.ghost) }.not_to change { User.ghost.reload.sign_in_count }
end
end
diff --git a/spec/features/projects/members/sorting_spec.rb b/spec/features/projects/members/sorting_spec.rb
index afb613f034e74acfecc6dee2475a4ec8f7be1753..dc7236fa12091997f86ebafb8f39df5b0d43aad6 100644
--- a/spec/features/projects/members/sorting_spec.rb
+++ b/spec/features/projects/members/sorting_spec.rb
@@ -67,7 +67,7 @@
expect(page).to have_css('.member-sort-dropdown .dropdown-toggle-text', text: 'Name, descending')
end
- scenario 'sorts by recent sign in', :redis do
+ scenario 'sorts by recent sign in', :clean_gitlab_redis_shared_state do
visit_members_list(sort: :recent_sign_in)
expect(first_member).to include(master.name)
@@ -75,7 +75,7 @@
expect(page).to have_css('.member-sort-dropdown .dropdown-toggle-text', text: 'Recent sign in')
end
- scenario 'sorts by oldest sign in', :redis do
+ scenario 'sorts by oldest sign in', :clean_gitlab_redis_shared_state do
visit_members_list(sort: :oldest_sign_in)
expect(first_member).to include(developer.name)
diff --git a/spec/fixtures/config/redis_cache_config_with_env.yml b/spec/fixtures/config/redis_cache_config_with_env.yml
new file mode 100644
index 0000000000000000000000000000000000000000..52fd5a0646058598216841f033de2a6bb1f3c622
--- /dev/null
+++ b/spec/fixtures/config/redis_cache_config_with_env.yml
@@ -0,0 +1,2 @@
+test:
+ url: <%= ENV['TEST_GITLAB_REDIS_CACHE_URL'] %>
diff --git a/spec/fixtures/config/redis_cache_new_format_host.yml b/spec/fixtures/config/redis_cache_new_format_host.yml
new file mode 100644
index 0000000000000000000000000000000000000000..a24f3716391d5cfdf4c7ce34b3e5c1296940aaee
--- /dev/null
+++ b/spec/fixtures/config/redis_cache_new_format_host.yml
@@ -0,0 +1,29 @@
+# redis://[:password@]host[:port][/db-number][?option=value]
+# more details: http://www.iana.org/assignments/uri-schemes/prov/redis
+development:
+ url: redis://:mynewpassword@localhost:6380/10
+ sentinels:
+ -
+ host: localhost
+ port: 26380 # point to sentinel, not to redis port
+ -
+ host: slave2
+ port: 26380 # point to sentinel, not to redis port
+test:
+ url: redis://:mynewpassword@localhost:6380/10
+ sentinels:
+ -
+ host: localhost
+ port: 26380 # point to sentinel, not to redis port
+ -
+ host: slave2
+ port: 26380 # point to sentinel, not to redis port
+production:
+ url: redis://:mynewpassword@localhost:6380/10
+ sentinels:
+ -
+ host: slave1
+ port: 26380 # point to sentinel, not to redis port
+ -
+ host: slave2
+ port: 26380 # point to sentinel, not to redis port
diff --git a/spec/fixtures/config/redis_cache_new_format_socket.yml b/spec/fixtures/config/redis_cache_new_format_socket.yml
new file mode 100644
index 0000000000000000000000000000000000000000..3634c5501633365fb7eef70655922d36396e7d71
--- /dev/null
+++ b/spec/fixtures/config/redis_cache_new_format_socket.yml
@@ -0,0 +1,6 @@
+development:
+ url: unix:/path/to/redis.cache.sock
+test:
+ url: unix:/path/to/redis.cache.sock
+production:
+ url: unix:/path/to/redis.cache.sock
diff --git a/spec/fixtures/config/redis_cache_old_format_host.yml b/spec/fixtures/config/redis_cache_old_format_host.yml
new file mode 100644
index 0000000000000000000000000000000000000000..3609dcd022e57fdc57f7bfe472f8d5645dd63c26
--- /dev/null
+++ b/spec/fixtures/config/redis_cache_old_format_host.yml
@@ -0,0 +1,5 @@
+# redis://[:password@]host[:port][/db-number][?option=value]
+# more details: http://www.iana.org/assignments/uri-schemes/prov/redis
+development: redis://:mypassword@localhost:6380/10
+test: redis://:mypassword@localhost:6380/10
+production: redis://:mypassword@localhost:6380/10
diff --git a/spec/fixtures/config/redis_cache_old_format_socket.yml b/spec/fixtures/config/redis_cache_old_format_socket.yml
new file mode 100644
index 0000000000000000000000000000000000000000..26fa0eda2456f5b667f0b3bec890a247081455cd
--- /dev/null
+++ b/spec/fixtures/config/redis_cache_old_format_socket.yml
@@ -0,0 +1,3 @@
+development: unix:/path/to/old/redis.cache.sock
+test: unix:/path/to/old/redis.cache.sock
+production: unix:/path/to/old/redis.cache.sock
diff --git a/spec/fixtures/config/redis_new_format_host.yml b/spec/fixtures/config/redis_new_format_host.yml
index 13772677a45c2e837656e065ca272baed8858aac..8d134d467e9baaebb985b7f7dee7842afecd07be 100644
--- a/spec/fixtures/config/redis_new_format_host.yml
+++ b/spec/fixtures/config/redis_new_format_host.yml
@@ -5,25 +5,25 @@ development:
sentinels:
-
host: localhost
- port: 26380 # point to sentinel, not to redis port
+ port: 26379 # point to sentinel, not to redis port
-
host: slave2
- port: 26381 # point to sentinel, not to redis port
+ port: 26379 # point to sentinel, not to redis port
test:
url: redis://:mynewpassword@localhost:6379/99
sentinels:
-
host: localhost
- port: 26380 # point to sentinel, not to redis port
+ port: 26379 # point to sentinel, not to redis port
-
host: slave2
- port: 26381 # point to sentinel, not to redis port
+ port: 26379 # point to sentinel, not to redis port
production:
url: redis://:mynewpassword@localhost:6379/99
sentinels:
-
host: slave1
- port: 26380 # point to sentinel, not to redis port
+ port: 26379 # point to sentinel, not to redis port
-
host: slave2
- port: 26381 # point to sentinel, not to redis port
+ port: 26379 # point to sentinel, not to redis port
diff --git a/spec/fixtures/config/redis_queues_config_with_env.yml b/spec/fixtures/config/redis_queues_config_with_env.yml
new file mode 100644
index 0000000000000000000000000000000000000000..d16a9d8a7f85b7c930d72064d317b15521e621b0
--- /dev/null
+++ b/spec/fixtures/config/redis_queues_config_with_env.yml
@@ -0,0 +1,2 @@
+test:
+ url: <%= ENV['TEST_GITLAB_REDIS_QUEUES_URL'] %>
diff --git a/spec/fixtures/config/redis_queues_new_format_host.yml b/spec/fixtures/config/redis_queues_new_format_host.yml
new file mode 100644
index 0000000000000000000000000000000000000000..1535584d77995ff6741e83a7793094be2ea3519f
--- /dev/null
+++ b/spec/fixtures/config/redis_queues_new_format_host.yml
@@ -0,0 +1,29 @@
+# redis://[:password@]host[:port][/db-number][?option=value]
+# more details: http://www.iana.org/assignments/uri-schemes/prov/redis
+development:
+ url: redis://:mynewpassword@localhost:6381/11
+ sentinels:
+ -
+ host: localhost
+ port: 26381 # point to sentinel, not to redis port
+ -
+ host: slave2
+ port: 26381 # point to sentinel, not to redis port
+test:
+ url: redis://:mynewpassword@localhost:6381/11
+ sentinels:
+ -
+ host: localhost
+ port: 26381 # point to sentinel, not to redis port
+ -
+ host: slave2
+ port: 26381 # point to sentinel, not to redis port
+production:
+ url: redis://:mynewpassword@localhost:6381/11
+ sentinels:
+ -
+ host: slave1
+ port: 26381 # point to sentinel, not to redis port
+ -
+ host: slave2
+ port: 26381 # point to sentinel, not to redis port
diff --git a/spec/fixtures/config/redis_queues_new_format_socket.yml b/spec/fixtures/config/redis_queues_new_format_socket.yml
new file mode 100644
index 0000000000000000000000000000000000000000..b488d84d022a5ece220705635b3a4077be55ccba
--- /dev/null
+++ b/spec/fixtures/config/redis_queues_new_format_socket.yml
@@ -0,0 +1,6 @@
+development:
+ url: unix:/path/to/redis.queues.sock
+test:
+ url: unix:/path/to/redis.queues.sock
+production:
+ url: unix:/path/to/redis.queues.sock
diff --git a/spec/fixtures/config/redis_queues_old_format_host.yml b/spec/fixtures/config/redis_queues_old_format_host.yml
new file mode 100644
index 0000000000000000000000000000000000000000..6531748a8d78ed4e79ec16c5f93619088e52af91
--- /dev/null
+++ b/spec/fixtures/config/redis_queues_old_format_host.yml
@@ -0,0 +1,5 @@
+# redis://[:password@]host[:port][/db-number][?option=value]
+# more details: http://www.iana.org/assignments/uri-schemes/prov/redis
+development: redis://:mypassword@localhost:6381/11
+test: redis://:mypassword@localhost:6381/11
+production: redis://:mypassword@localhost:6381/11
diff --git a/spec/fixtures/config/redis_queues_old_format_socket.yml b/spec/fixtures/config/redis_queues_old_format_socket.yml
new file mode 100644
index 0000000000000000000000000000000000000000..53f5db727587bbea7b9d7b434b3472dd403627b0
--- /dev/null
+++ b/spec/fixtures/config/redis_queues_old_format_socket.yml
@@ -0,0 +1,3 @@
+development: unix:/path/to/old/redis.queues.sock
+test: unix:/path/to/old/redis.queues.sock
+production: unix:/path/to/old/redis.queues.sock
diff --git a/spec/fixtures/config/redis_shared_state_config_with_env.yml b/spec/fixtures/config/redis_shared_state_config_with_env.yml
new file mode 100644
index 0000000000000000000000000000000000000000..eab7203d0de50daff598f417146d1e30dd2e0423
--- /dev/null
+++ b/spec/fixtures/config/redis_shared_state_config_with_env.yml
@@ -0,0 +1,2 @@
+test:
+ url: <%= ENV['TEST_GITLAB_REDIS_SHARED_STATE_URL'] %>
diff --git a/spec/fixtures/config/redis_shared_state_new_format_host.yml b/spec/fixtures/config/redis_shared_state_new_format_host.yml
new file mode 100644
index 0000000000000000000000000000000000000000..1180b2b4a82055e34bb6edc9ff38796f7ffe5bbe
--- /dev/null
+++ b/spec/fixtures/config/redis_shared_state_new_format_host.yml
@@ -0,0 +1,29 @@
+# redis://[:password@]host[:port][/db-number][?option=value]
+# more details: http://www.iana.org/assignments/uri-schemes/prov/redis
+development:
+ url: redis://:mynewpassword@localhost:6382/12
+ sentinels:
+ -
+ host: localhost
+ port: 26382 # point to sentinel, not to redis port
+ -
+ host: slave2
+ port: 26382 # point to sentinel, not to redis port
+test:
+ url: redis://:mynewpassword@localhost:6382/12
+ sentinels:
+ -
+ host: localhost
+ port: 26382 # point to sentinel, not to redis port
+ -
+ host: slave2
+ port: 26382 # point to sentinel, not to redis port
+production:
+ url: redis://:mynewpassword@localhost:6382/12
+ sentinels:
+ -
+ host: slave1
+ port: 26382 # point to sentinel, not to redis port
+ -
+ host: slave2
+ port: 26382 # point to sentinel, not to redis port
diff --git a/spec/fixtures/config/redis_shared_state_new_format_socket.yml b/spec/fixtures/config/redis_shared_state_new_format_socket.yml
new file mode 100644
index 0000000000000000000000000000000000000000..1b0e699729eb72b3ca1e776cd1cd10883b8d7e65
--- /dev/null
+++ b/spec/fixtures/config/redis_shared_state_new_format_socket.yml
@@ -0,0 +1,6 @@
+development:
+ url: unix:/path/to/redis.shared_state.sock
+test:
+ url: unix:/path/to/redis.shared_state.sock
+production:
+ url: unix:/path/to/redis.shared_state.sock
diff --git a/spec/fixtures/config/redis_shared_state_old_format_host.yml b/spec/fixtures/config/redis_shared_state_old_format_host.yml
new file mode 100644
index 0000000000000000000000000000000000000000..fef5e768c5db9a24d882686a0caaba8ac752c3b1
--- /dev/null
+++ b/spec/fixtures/config/redis_shared_state_old_format_host.yml
@@ -0,0 +1,5 @@
+# redis://[:password@]host[:port][/db-number][?option=value]
+# more details: http://www.iana.org/assignments/uri-schemes/prov/redis
+development: redis://:mypassword@localhost:6382/12
+test: redis://:mypassword@localhost:6382/12
+production: redis://:mypassword@localhost:6382/12
diff --git a/spec/fixtures/config/redis_shared_state_old_format_socket.yml b/spec/fixtures/config/redis_shared_state_old_format_socket.yml
new file mode 100644
index 0000000000000000000000000000000000000000..4746afbb5ef5223e930201ae36840e2e9ccd008c
--- /dev/null
+++ b/spec/fixtures/config/redis_shared_state_old_format_socket.yml
@@ -0,0 +1,3 @@
+development: unix:/path/to/old/redis.shared_state.sock
+test: unix:/path/to/old/redis.shared_state.sock
+production: unix:/path/to/old/redis.shared_state.sock
diff --git a/spec/helpers/issuables_helper_spec.rb b/spec/helpers/issuables_helper_spec.rb
index d2e918ef0141aa0c9eba4d5702c0b03f206bd1af..b423a09873bde5e745544c63c56ea8c23767ef3b 100644
--- a/spec/helpers/issuables_helper_spec.rb
+++ b/spec/helpers/issuables_helper_spec.rb
@@ -60,7 +60,7 @@
end
end
- describe 'counter caching based on issuable type and params', :caching do
+ describe 'counter caching based on issuable type and params', :use_clean_rails_memory_store_caching do
let(:params) do
{
scope: 'created-by-me',
diff --git a/spec/helpers/projects_helper_spec.rb b/spec/helpers/projects_helper_spec.rb
index 487d980070715604f58a3b3eb9fe20b801c5a7aa..c462d9006ea5d15ca0797fb70d5deee3e8ccdce6 100644
--- a/spec/helpers/projects_helper_spec.rb
+++ b/spec/helpers/projects_helper_spec.rb
@@ -63,7 +63,7 @@
end
end
- describe "#project_list_cache_key", redis: true do
+ describe "#project_list_cache_key", clean_gitlab_redis_shared_state: true do
let(:project) { create(:project) }
it "includes the route" do
diff --git a/spec/lib/gitlab/auth/unique_ips_limiter_spec.rb b/spec/lib/gitlab/auth/unique_ips_limiter_spec.rb
index fc72df575be124dc694c3dabe3ffef62daaa83b1..15b3db0ed3db8ab4e3a20532b0bc003d6d906782 100644
--- a/spec/lib/gitlab/auth/unique_ips_limiter_spec.rb
+++ b/spec/lib/gitlab/auth/unique_ips_limiter_spec.rb
@@ -1,6 +1,6 @@
require 'spec_helper'
-describe Gitlab::Auth::UniqueIpsLimiter, :redis, lib: true do
+describe Gitlab::Auth::UniqueIpsLimiter, :clean_gitlab_redis_shared_state, lib: true do
include_context 'unique ips sign in limit'
let(:user) { create(:user) }
diff --git a/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb b/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb
index 07db6c3a640bbf8273fd6a8051f4b958781a3f9f..0daf41a7c8639f6abf46cf7c6150d50ff9e91e4d 100644
--- a/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb
+++ b/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb
@@ -1,6 +1,6 @@
require 'spec_helper'
-describe Gitlab::Cache::Ci::ProjectPipelineStatus, :redis do
+describe Gitlab::Cache::Ci::ProjectPipelineStatus, :clean_gitlab_redis_cache do
let!(:project) { create(:project) }
let(:pipeline_status) { described_class.new(project) }
let(:cache_key) { "projects/#{project.id}/pipeline_status" }
@@ -28,8 +28,8 @@
expect(project.instance_variable_get('@pipeline_status')).to be_a(described_class)
end
- describe 'without a status in redis' do
- it 'loads the status from a commit when it was not in redis' do
+ describe 'without a status in redis_cache' do
+ it 'loads the status from a commit when it was not in redis_cache' do
empty_status = { sha: nil, status: nil, ref: nil }
fake_pipeline = described_class.new(
project_without_status,
@@ -48,9 +48,9 @@
described_class.load_in_batch_for_projects([project_without_status])
end
- it 'only connects to redis twice' do
+ it 'only connects to redis_cache twice' do
# Once to load, once to store in the cache
- expect(Gitlab::Redis).to receive(:with).exactly(2).and_call_original
+ expect(Gitlab::Redis::Cache).to receive(:with).exactly(2).and_call_original
described_class.load_in_batch_for_projects([project_without_status])
@@ -58,9 +58,9 @@
end
end
- describe 'when a status was cached in redis' do
+ describe 'when a status was cached in redis_cache' do
before do
- Gitlab::Redis.with do |redis|
+ Gitlab::Redis::Cache.with do |redis|
redis.mapped_hmset(cache_key,
{ sha: sha, status: status, ref: ref })
end
@@ -76,8 +76,8 @@
expect(pipeline_status.ref).to eq(ref)
end
- it 'only connects to redis once' do
- expect(Gitlab::Redis).to receive(:with).exactly(1).and_call_original
+ it 'only connects to redis_cache once' do
+ expect(Gitlab::Redis::Cache).to receive(:with).exactly(1).and_call_original
described_class.load_in_batch_for_projects([project])
@@ -94,8 +94,8 @@
end
describe '.cached_results_for_projects' do
- it 'loads a status from redis for all projects' do
- Gitlab::Redis.with do |redis|
+ it 'loads a status from caching for all projects' do
+ Gitlab::Redis::Cache.with do |redis|
redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref })
end
@@ -183,7 +183,7 @@
end
end
- describe "#load_from_project" do
+ describe "#load_from_project", :clean_gitlab_redis_cache do
let!(:pipeline) { create(:ci_pipeline, :success, project: project, sha: project.commit.sha) }
it 'reads the status from the pipeline for the commit' do
@@ -203,40 +203,40 @@
end
end
- describe "#store_in_cache", :redis do
- it "sets the object in redis" do
+ describe "#store_in_cache", :clean_gitlab_redis_cache do
+ it "sets the object in caching" do
pipeline_status.sha = '123456'
pipeline_status.status = 'failed'
pipeline_status.store_in_cache
- read_sha, read_status = Gitlab::Redis.with { |redis| redis.hmget(cache_key, :sha, :status) }
+ read_sha, read_status = Gitlab::Redis::Cache.with { |redis| redis.hmget(cache_key, :sha, :status) }
expect(read_sha).to eq('123456')
expect(read_status).to eq('failed')
end
end
- describe '#store_in_cache_if_needed', :redis do
+ describe '#store_in_cache_if_needed', :clean_gitlab_redis_cache do
it 'stores the state in the cache when the sha is the HEAD of the project' do
create(:ci_pipeline, :success, project: project, sha: project.commit.sha)
pipeline_status = described_class.load_for_project(project)
pipeline_status.store_in_cache_if_needed
- sha, status, ref = Gitlab::Redis.with { |redis| redis.hmget(cache_key, :sha, :status, :ref) }
+ sha, status, ref = Gitlab::Redis::Cache.with { |redis| redis.hmget(cache_key, :sha, :status, :ref) }
expect(sha).not_to be_nil
expect(status).not_to be_nil
expect(ref).not_to be_nil
end
- it "doesn't store the status in redis when the sha is not the head of the project" do
+ it "doesn't store the status in redis_cache when the sha is not the head of the project" do
other_status = described_class.new(
project,
pipeline_info: { sha: "123456", status: "failed" }
)
other_status.store_in_cache_if_needed
- sha, status = Gitlab::Redis.with { |redis| redis.hmget(cache_key, :sha, :status) }
+ sha, status = Gitlab::Redis::Cache.with { |redis| redis.hmget(cache_key, :sha, :status) }
expect(sha).to be_nil
expect(status).to be_nil
@@ -244,7 +244,7 @@
it "deletes the cache if the repository doesn't have a head commit" do
empty_project = create(:empty_project)
- Gitlab::Redis.with do |redis|
+ Gitlab::Redis::Cache.with do |redis|
redis.mapped_hmset(cache_key,
{ sha: 'sha', status: 'pending', ref: 'master' })
end
@@ -255,7 +255,7 @@
})
other_status.store_in_cache_if_needed
- sha, status, ref = Gitlab::Redis.with { |redis| redis.hmget("projects/#{empty_project.id}/pipeline_status", :sha, :status, :ref) }
+ sha, status, ref = Gitlab::Redis::Cache.with { |redis| redis.hmget("projects/#{empty_project.id}/pipeline_status", :sha, :status, :ref) }
expect(sha).to be_nil
expect(status).to be_nil
@@ -263,20 +263,20 @@
end
end
- describe "with a status in redis", :redis do
+ describe "with a status in caching", :clean_gitlab_redis_cache do
let(:status) { 'success' }
let(:sha) { '424d1b73bc0d3cb726eb7dc4ce17a4d48552f8c6' }
let(:ref) { 'master' }
before do
- Gitlab::Redis.with do |redis|
+ Gitlab::Redis::Cache.with do |redis|
redis.mapped_hmset(cache_key,
{ sha: sha, status: status, ref: ref })
end
end
describe '#load_from_cache' do
- it 'reads the status from redis' do
+ it 'reads the status from redis_cache' do
pipeline_status.load_from_cache
expect(pipeline_status.sha).to eq(sha)
@@ -292,10 +292,10 @@
end
describe '#delete_from_cache' do
- it 'deletes values from redis' do
+ it 'deletes values from redis_cache' do
pipeline_status.delete_from_cache
- key_exists = Gitlab::Redis.with { |redis| redis.exists(cache_key) }
+ key_exists = Gitlab::Redis::Cache.with { |redis| redis.exists(cache_key) }
expect(key_exists).to be_falsy
end
diff --git a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base_spec.rb b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base_spec.rb
index 8813f129ef5d0bd329403b8d7fc4951cf04a4207..df7d1b5d27a082d5080459b556ba0e8d68a7f28f 100644
--- a/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base_spec.rb
+++ b/spec/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base_spec.rb
@@ -236,7 +236,7 @@ def migration_project(project)
subject.track_rename('namespace', 'path/to/namespace', 'path/to/renamed')
old_path, new_path = [nil, nil]
- Gitlab::Redis.with do |redis|
+ Gitlab::Redis::SharedState.with do |redis|
rename_info = redis.lpop(key)
old_path, new_path = JSON.parse(rename_info)
end
@@ -268,7 +268,7 @@ def migration_project(project)
key = 'rename:FakeRenameReservedPathMigrationV1:project'
stored_renames = nil
rename_count = 0
- Gitlab::Redis.with do |redis|
+ Gitlab::Redis::SharedState.with do |redis|
stored_renames = redis.lrange(key, 0, 1)
rename_count = redis.llen(key)
end
diff --git a/spec/lib/gitlab/exclusive_lease_spec.rb b/spec/lib/gitlab/exclusive_lease_spec.rb
index 81bbd70ffb8c78574b1d9f7105dbf57ea1601aeb..590d6da41132889d4c35386c68a7aaad4f76c3a8 100644
--- a/spec/lib/gitlab/exclusive_lease_spec.rb
+++ b/spec/lib/gitlab/exclusive_lease_spec.rb
@@ -1,6 +1,6 @@
require 'spec_helper'
-describe Gitlab::ExclusiveLease, type: :redis do
+describe Gitlab::ExclusiveLease, type: :clean_gitlab_redis_shared_state do
let(:unique_key) { SecureRandom.hex(10) }
describe '#try_obtain' do
diff --git a/spec/lib/gitlab/health_checks/redis/cache_check_spec.rb b/spec/lib/gitlab/health_checks/redis/cache_check_spec.rb
new file mode 100644
index 0000000000000000000000000000000000000000..3693f52b51b75c1caa326207f1f07f1e177f87de
--- /dev/null
+++ b/spec/lib/gitlab/health_checks/redis/cache_check_spec.rb
@@ -0,0 +1,6 @@
+require 'spec_helper'
+require_relative '../simple_check_shared'
+
+describe Gitlab::HealthChecks::Redis::CacheCheck do
+ include_examples 'simple_check', 'redis_cache_ping', 'RedisCache', 'PONG'
+end
diff --git a/spec/lib/gitlab/health_checks/redis/queues_check_spec.rb b/spec/lib/gitlab/health_checks/redis/queues_check_spec.rb
new file mode 100644
index 0000000000000000000000000000000000000000..c69443d205d0934b47c5197e3dce33a0e11ee58c
--- /dev/null
+++ b/spec/lib/gitlab/health_checks/redis/queues_check_spec.rb
@@ -0,0 +1,6 @@
+require 'spec_helper'
+require_relative '../simple_check_shared'
+
+describe Gitlab::HealthChecks::Redis::QueuesCheck do
+ include_examples 'simple_check', 'redis_queues_ping', 'RedisQueues', 'PONG'
+end
diff --git a/spec/lib/gitlab/health_checks/redis/redis_check_spec.rb b/spec/lib/gitlab/health_checks/redis/redis_check_spec.rb
new file mode 100644
index 0000000000000000000000000000000000000000..03afc1cd761ced8a9657f278dc5e61f67ffc83ba
--- /dev/null
+++ b/spec/lib/gitlab/health_checks/redis/redis_check_spec.rb
@@ -0,0 +1,6 @@
+require 'spec_helper'
+require_relative '../simple_check_shared'
+
+describe Gitlab::HealthChecks::Redis::RedisCheck do
+ include_examples 'simple_check', 'redis_ping', 'Redis', 'PONG'
+end
diff --git a/spec/lib/gitlab/health_checks/redis/shared_state_check_spec.rb b/spec/lib/gitlab/health_checks/redis/shared_state_check_spec.rb
new file mode 100644
index 0000000000000000000000000000000000000000..b72e152bbe2f8d0eca10b52b39407ecd022c3f53
--- /dev/null
+++ b/spec/lib/gitlab/health_checks/redis/shared_state_check_spec.rb
@@ -0,0 +1,6 @@
+require 'spec_helper'
+require_relative '../simple_check_shared'
+
+describe Gitlab::HealthChecks::Redis::SharedStateCheck do
+ include_examples 'simple_check', 'redis_shared_state_ping', 'RedisSharedState', 'PONG'
+end
diff --git a/spec/lib/gitlab/health_checks/redis_check_spec.rb b/spec/lib/gitlab/health_checks/redis_check_spec.rb
deleted file mode 100644
index 734cdcb893e56c87f3d17803af8ccbf3cb139fa2..0000000000000000000000000000000000000000
--- a/spec/lib/gitlab/health_checks/redis_check_spec.rb
+++ /dev/null
@@ -1,6 +0,0 @@
-require 'spec_helper'
-require_relative './simple_check_shared'
-
-describe Gitlab::HealthChecks::RedisCheck do
- include_examples 'simple_check', 'redis_ping', 'Redis', 'PONG'
-end
diff --git a/spec/lib/gitlab/health_checks/simple_check_shared.rb b/spec/lib/gitlab/health_checks/simple_check_shared.rb
index 3f871d6603442aab9438f53be98b3fb6f2198328..1abebeac4dddd7e190e013ba5762fd049ddac428 100644
--- a/spec/lib/gitlab/health_checks/simple_check_shared.rb
+++ b/spec/lib/gitlab/health_checks/simple_check_shared.rb
@@ -47,7 +47,7 @@
allow(described_class).to receive(:check).and_return 'error!'
end
- it { is_expected.to have_attributes(success: false, message: "unexpected #{check_name} check result: error!") }
+ it { is_expected.to have_attributes(success: false, message: "unexpected #{described_class.human_name} check result: error!") }
end
context 'Check is timeouting' do
@@ -55,7 +55,7 @@
allow(described_class).to receive(:check ).and_return Timeout::Error.new
end
- it { is_expected.to have_attributes(success: false, message: "#{check_name} check timed out") }
+ it { is_expected.to have_attributes(success: false, message: "#{described_class.human_name} check timed out") }
end
end
diff --git a/spec/lib/gitlab/performance_bar_spec.rb b/spec/lib/gitlab/performance_bar_spec.rb
index 8a586bdbf630658c6cdad3915a6943f6003a29e7..b8a2267f1a4f2655f5db4889ef86f170d505278e 100644
--- a/spec/lib/gitlab/performance_bar_spec.rb
+++ b/spec/lib/gitlab/performance_bar_spec.rb
@@ -7,7 +7,7 @@
described_class.enabled?(user)
end
- it 'caches the allowed user IDs in cache', :caching do
+ it 'caches the allowed user IDs in cache', :use_clean_rails_memory_store_caching do
expect do
expect(described_class.enabled?(user)).to be_truthy
end.not_to exceed_query_limit(0)
diff --git a/spec/lib/gitlab/redis/cache_spec.rb b/spec/lib/gitlab/redis/cache_spec.rb
new file mode 100644
index 0000000000000000000000000000000000000000..5a4f17cfcf6aba7ace7494d741244bea4225d6d9
--- /dev/null
+++ b/spec/lib/gitlab/redis/cache_spec.rb
@@ -0,0 +1,20 @@
+require 'spec_helper'
+
+describe Gitlab::Redis::Cache do
+ let(:config_file_name) { "config/redis.cache.yml" }
+ let(:environment_config_file_name) { "GITLAB_REDIS_CACHE_CONFIG_FILE" }
+ let(:config_old_format_socket) { "spec/fixtures/config/redis_cache_old_format_socket.yml" }
+ let(:config_new_format_socket) { "spec/fixtures/config/redis_cache_new_format_socket.yml" }
+ let(:old_socket_path) {"/path/to/old/redis.cache.sock" }
+ let(:new_socket_path) {"/path/to/redis.cache.sock" }
+ let(:config_old_format_host) { "spec/fixtures/config/redis_cache_old_format_host.yml" }
+ let(:config_new_format_host) { "spec/fixtures/config/redis_cache_new_format_host.yml" }
+ let(:redis_port) { 6380 }
+ let(:redis_database) { 10 }
+ let(:sentinel_port) { redis_port + 20000 }
+ let(:config_with_environment_variable_inside) { "spec/fixtures/config/redis_cache_config_with_env.yml"}
+ let(:config_env_variable_url) {"TEST_GITLAB_REDIS_CACHE_URL"}
+ let(:class_redis_url) { Gitlab::Redis::Cache::DEFAULT_REDIS_CACHE_URL }
+
+ include_examples "redis_shared_examples"
+end
diff --git a/spec/lib/gitlab/redis/queues_spec.rb b/spec/lib/gitlab/redis/queues_spec.rb
new file mode 100644
index 0000000000000000000000000000000000000000..01ca25635a98cecbf36276b7c328e1c5ad7e27df
--- /dev/null
+++ b/spec/lib/gitlab/redis/queues_spec.rb
@@ -0,0 +1,20 @@
+require 'spec_helper'
+
+describe Gitlab::Redis::Queues do
+ let(:config_file_name) { "config/redis.queues.yml" }
+ let(:environment_config_file_name) { "GITLAB_REDIS_QUEUES_CONFIG_FILE" }
+ let(:config_old_format_socket) { "spec/fixtures/config/redis_queues_old_format_socket.yml" }
+ let(:config_new_format_socket) { "spec/fixtures/config/redis_queues_new_format_socket.yml" }
+ let(:old_socket_path) {"/path/to/old/redis.queues.sock" }
+ let(:new_socket_path) {"/path/to/redis.queues.sock" }
+ let(:config_old_format_host) { "spec/fixtures/config/redis_queues_old_format_host.yml" }
+ let(:config_new_format_host) { "spec/fixtures/config/redis_queues_new_format_host.yml" }
+ let(:redis_port) { 6381 }
+ let(:redis_database) { 11 }
+ let(:sentinel_port) { redis_port + 20000 }
+ let(:config_with_environment_variable_inside) { "spec/fixtures/config/redis_queues_config_with_env.yml"}
+ let(:config_env_variable_url) {"TEST_GITLAB_REDIS_QUEUES_URL"}
+ let(:class_redis_url) { Gitlab::Redis::Queues::DEFAULT_REDIS_QUEUES_URL }
+
+ include_examples "redis_shared_examples"
+end
diff --git a/spec/lib/gitlab/redis/shared_state_spec.rb b/spec/lib/gitlab/redis/shared_state_spec.rb
new file mode 100644
index 0000000000000000000000000000000000000000..24b73745dc5227d68d97ed4f40d0b803fa5dd7b2
--- /dev/null
+++ b/spec/lib/gitlab/redis/shared_state_spec.rb
@@ -0,0 +1,20 @@
+require 'spec_helper'
+
+describe Gitlab::Redis::SharedState do
+ let(:config_file_name) { "config/redis.shared_state.yml" }
+ let(:environment_config_file_name) { "GITLAB_REDIS_SHARED_STATE_CONFIG_FILE" }
+ let(:config_old_format_socket) { "spec/fixtures/config/redis_shared_state_old_format_socket.yml" }
+ let(:config_new_format_socket) { "spec/fixtures/config/redis_shared_state_new_format_socket.yml" }
+ let(:old_socket_path) {"/path/to/old/redis.shared_state.sock" }
+ let(:new_socket_path) {"/path/to/redis.shared_state.sock" }
+ let(:config_old_format_host) { "spec/fixtures/config/redis_shared_state_old_format_host.yml" }
+ let(:config_new_format_host) { "spec/fixtures/config/redis_shared_state_new_format_host.yml" }
+ let(:redis_port) { 6382 }
+ let(:redis_database) { 12 }
+ let(:sentinel_port) { redis_port + 20000 }
+ let(:config_with_environment_variable_inside) { "spec/fixtures/config/redis_shared_state_config_with_env.yml"}
+ let(:config_env_variable_url) {"TEST_GITLAB_REDIS_SHARED_STATE_URL"}
+ let(:class_redis_url) { Gitlab::Redis::SharedState::DEFAULT_REDIS_SHARED_STATE_URL }
+
+ include_examples "redis_shared_examples"
+end
diff --git a/spec/lib/gitlab/redis/wrapper_spec.rb b/spec/lib/gitlab/redis/wrapper_spec.rb
new file mode 100644
index 0000000000000000000000000000000000000000..e1becd0a6145c5681b7d68f28b8bdf472cf183d7
--- /dev/null
+++ b/spec/lib/gitlab/redis/wrapper_spec.rb
@@ -0,0 +1,20 @@
+require 'spec_helper'
+
+describe Gitlab::Redis::Wrapper do
+ let(:config_file_name) { "config/resque.yml" }
+ let(:environment_config_file_name) { "GITLAB_REDIS_CONFIG_FILE" }
+ let(:config_old_format_socket) { "spec/fixtures/config/redis_old_format_socket.yml" }
+ let(:config_new_format_socket) { "spec/fixtures/config/redis_new_format_socket.yml" }
+ let(:old_socket_path) {"/path/to/old/redis.sock" }
+ let(:new_socket_path) {"/path/to/redis.sock" }
+ let(:config_old_format_host) { "spec/fixtures/config/redis_old_format_host.yml" }
+ let(:config_new_format_host) { "spec/fixtures/config/redis_new_format_host.yml" }
+ let(:redis_port) { 6379 }
+ let(:redis_database) { 99 }
+ let(:sentinel_port) { redis_port + 20000 }
+ let(:config_with_environment_variable_inside) { "spec/fixtures/config/redis_config_with_env.yml"}
+ let(:config_env_variable_url) {"TEST_GITLAB_REDIS_URL"}
+ let(:class_redis_url) { Gitlab::Redis::Wrapper::DEFAULT_REDIS_URL }
+
+ include_examples "redis_shared_examples"
+end
diff --git a/spec/lib/gitlab/sidekiq_status_spec.rb b/spec/lib/gitlab/sidekiq_status_spec.rb
index 496e50fbae4bff09b77c8b9a5bf7dca29583c276..c2e77ef6b6c41629eb24d5c152c76f5e79469bd1 100644
--- a/spec/lib/gitlab/sidekiq_status_spec.rb
+++ b/spec/lib/gitlab/sidekiq_status_spec.rb
@@ -1,7 +1,7 @@
require 'spec_helper'
describe Gitlab::SidekiqStatus do
- describe '.set', :redis do
+ describe '.set', :clean_gitlab_redis_shared_state do
it 'stores the job ID' do
described_class.set('123')
@@ -14,7 +14,7 @@
end
end
- describe '.unset', :redis do
+ describe '.unset', :clean_gitlab_redis_shared_state do
it 'removes the job ID' do
described_class.set('123')
described_class.unset('123')
@@ -27,7 +27,7 @@
end
end
- describe '.all_completed?', :redis do
+ describe '.all_completed?', :clean_gitlab_redis_shared_state do
it 'returns true if all jobs have been completed' do
expect(described_class.all_completed?(%w(123))).to eq(true)
end
@@ -39,7 +39,7 @@
end
end
- describe '.num_running', :redis do
+ describe '.num_running', :clean_gitlab_redis_shared_state do
it 'returns 0 if all jobs have been completed' do
expect(described_class.num_running(%w(123))).to eq(0)
end
@@ -52,7 +52,7 @@
end
end
- describe '.num_completed', :redis do
+ describe '.num_completed', :clean_gitlab_redis_shared_state do
it 'returns 1 if all jobs have been completed' do
expect(described_class.num_completed(%w(123))).to eq(1)
end
@@ -74,7 +74,7 @@
end
end
- describe 'completed', :redis do
+ describe 'completed', :clean_gitlab_redis_shared_state do
it 'returns the completed job' do
expect(described_class.completed_jids(%w(123))).to eq(['123'])
end
diff --git a/spec/lib/gitlab/user_activities_spec.rb b/spec/lib/gitlab/user_activities_spec.rb
index 187d88c8c588d36b325a565f365c8c1967607707..a4ea0ac59e965fe75ab8073c8594668b61b76606 100644
--- a/spec/lib/gitlab/user_activities_spec.rb
+++ b/spec/lib/gitlab/user_activities_spec.rb
@@ -1,27 +1,27 @@
require 'spec_helper'
-describe Gitlab::UserActivities, :redis, lib: true do
+describe Gitlab::UserActivities, :clean_gitlab_redis_shared_state, lib: true do
let(:now) { Time.now }
describe '.record' do
context 'with no time given' do
- it 'uses Time.now and records an activity in Redis' do
+ it 'uses Time.now and records an activity in SharedState' do
Timecop.freeze do
now # eager-load now
described_class.record(42)
end
- Gitlab::Redis.with do |redis|
+ Gitlab::Redis::SharedState.with do |redis|
expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]])
end
end
end
context 'with a time given' do
- it 'uses the given time and records an activity in Redis' do
+ it 'uses the given time and records an activity in SharedState' do
described_class.record(42, now)
- Gitlab::Redis.with do |redis|
+ Gitlab::Redis::SharedState.with do |redis|
expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]])
end
end
@@ -31,30 +31,30 @@
describe '.delete' do
context 'with a single key' do
context 'and key exists' do
- it 'removes the pair from Redis' do
+ it 'removes the pair from SharedState' do
described_class.record(42, now)
- Gitlab::Redis.with do |redis|
+ Gitlab::Redis::SharedState.with do |redis|
expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]])
end
subject.delete(42)
- Gitlab::Redis.with do |redis|
+ Gitlab::Redis::SharedState.with do |redis|
expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []])
end
end
end
context 'and key does not exist' do
- it 'removes the pair from Redis' do
- Gitlab::Redis.with do |redis|
+ it 'removes the pair from SharedState' do
+ Gitlab::Redis::SharedState.with do |redis|
expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []])
end
subject.delete(42)
- Gitlab::Redis.with do |redis|
+ Gitlab::Redis::SharedState.with do |redis|
expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []])
end
end
@@ -63,33 +63,33 @@
context 'with multiple keys' do
context 'and all keys exist' do
- it 'removes the pair from Redis' do
+ it 'removes the pair from SharedState' do
described_class.record(41, now)
described_class.record(42, now)
- Gitlab::Redis.with do |redis|
+ Gitlab::Redis::SharedState.with do |redis|
expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['41', now.to_i.to_s], ['42', now.to_i.to_s]]])
end
subject.delete(41, 42)
- Gitlab::Redis.with do |redis|
+ Gitlab::Redis::SharedState.with do |redis|
expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []])
end
end
end
context 'and some keys does not exist' do
- it 'removes the existing pair from Redis' do
+ it 'removes the existing pair from SharedState' do
described_class.record(42, now)
- Gitlab::Redis.with do |redis|
+ Gitlab::Redis::SharedState.with do |redis|
expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]])
end
subject.delete(41, 42)
- Gitlab::Redis.with do |redis|
+ Gitlab::Redis::SharedState.with do |redis|
expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []])
end
end
diff --git a/spec/lib/gitlab/workhorse_spec.rb b/spec/lib/gitlab/workhorse_spec.rb
index 493ff3bb5fbc3605267f95208267acb7bc096cbb..efff0a152a8cedb76ae9abbf6ec964d0cd32a2a8 100644
--- a/spec/lib/gitlab/workhorse_spec.rb
+++ b/spec/lib/gitlab/workhorse_spec.rb
@@ -276,7 +276,7 @@ def call_verify(headers)
end
it 'set and notify' do
- expect_any_instance_of(Redis).to receive(:publish)
+ expect_any_instance_of(::Redis).to receive(:publish)
.with(described_class::NOTIFICATION_CHANNEL, "test-key=test-value")
subject
@@ -310,7 +310,7 @@ def call_verify(headers)
end
it 'does not notify' do
- expect_any_instance_of(Redis).not_to receive(:publish)
+ expect_any_instance_of(::Redis).not_to receive(:publish)
subject
end
diff --git a/spec/migrations/migrate_process_commit_worker_jobs_spec.rb b/spec/migrations/migrate_process_commit_worker_jobs_spec.rb
index 4223d2337a80432552e8545cd7422805f13d53ef..5b633dd349b79d352e16e788d45d8b866d7b9765 100644
--- a/spec/migrations/migrate_process_commit_worker_jobs_spec.rb
+++ b/spec/migrations/migrate_process_commit_worker_jobs_spec.rb
@@ -54,7 +54,7 @@
end
end
- describe '#up', :redis do
+ describe '#up', :clean_gitlab_redis_shared_state do
let(:migration) { described_class.new }
def job_count
@@ -172,7 +172,7 @@ def pop_job
end
end
- describe '#down', :redis do
+ describe '#down', :clean_gitlab_redis_shared_state do
let(:migration) { described_class.new }
def job_count
diff --git a/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb b/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb
index e3b42b5eac8baee3e6dc1d64836c8d43f29e4459..063829be5468778be6c2d8a4fd19a27733d2361d 100644
--- a/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb
+++ b/spec/migrations/migrate_user_activities_to_users_last_activity_on_spec.rb
@@ -3,13 +3,13 @@
require 'spec_helper'
require Rails.root.join('db', 'post_migrate', '20170324160416_migrate_user_activities_to_users_last_activity_on.rb')
-describe MigrateUserActivitiesToUsersLastActivityOn, :redis, :truncate do
+describe MigrateUserActivitiesToUsersLastActivityOn, :clean_gitlab_redis_shared_state, :truncate do
let(:migration) { described_class.new }
let!(:user_active_1) { create(:user) }
let!(:user_active_2) { create(:user) }
def record_activity(user, time)
- Gitlab::Redis.with do |redis|
+ Gitlab::Redis::SharedState.with do |redis|
redis.zadd(described_class::USER_ACTIVITY_SET_KEY, time.to_i, user.username)
end
end
diff --git a/spec/models/ci/runner_spec.rb b/spec/models/ci/runner_spec.rb
index 76ce558eea0a39e2e8d30ba579ee2a40880796ee..4b9cce28e0e68db30c32de9b1ac31abdf9561831 100644
--- a/spec/models/ci/runner_spec.rb
+++ b/spec/models/ci/runner_spec.rb
@@ -276,14 +276,14 @@
it 'sets a new last_update value when it is called the first time' do
last_update = runner.ensure_runner_queue_value
- expect_value_in_redis.to eq(last_update)
+ expect_value_in_queues.to eq(last_update)
end
it 'does not change if it is not expired and called again' do
last_update = runner.ensure_runner_queue_value
expect(runner.ensure_runner_queue_value).to eq(last_update)
- expect_value_in_redis.to eq(last_update)
+ expect_value_in_queues.to eq(last_update)
end
context 'updates runner queue after changing editable value' do
@@ -294,7 +294,7 @@
end
it 'sets a new last_update value' do
- expect_value_in_redis.not_to eq(last_update)
+ expect_value_in_queues.not_to eq(last_update)
end
end
@@ -306,12 +306,12 @@
end
it 'has an old last_update value' do
- expect_value_in_redis.to eq(last_update)
+ expect_value_in_queues.to eq(last_update)
end
end
- def expect_value_in_redis
- Gitlab::Redis.with do |redis|
+ def expect_value_in_queues
+ Gitlab::Redis::Queues.with do |redis|
runner_queue_key = runner.send(:runner_queue_key)
expect(redis.get(runner_queue_key))
end
@@ -330,7 +330,7 @@ def expect_value_in_redis
end
it 'cleans up the queue' do
- Gitlab::Redis.with do |redis|
+ Gitlab::Redis::Queues.with do |redis|
expect(redis.get(queue_key)).to be_nil
end
end
diff --git a/spec/models/concerns/reactive_caching_spec.rb b/spec/models/concerns/reactive_caching_spec.rb
index 808247ebfd53c327442fe563ff5c7c0020882090..5f9b7e0a367ffe3282deff9d0a26e501c6ca86cc 100644
--- a/spec/models/concerns/reactive_caching_spec.rb
+++ b/spec/models/concerns/reactive_caching_spec.rb
@@ -1,6 +1,6 @@
require 'spec_helper'
-describe ReactiveCaching, caching: true do
+describe ReactiveCaching, :use_clean_rails_memory_store_caching do
include ReactiveCachingHelpers
class CacheTest
diff --git a/spec/models/project_services/bamboo_service_spec.rb b/spec/models/project_services/bamboo_service_spec.rb
index 7b1a554d1fb53b15665b6b02bb4c49c798d31b97..99190d763f266d19ae9de81e4cc62f71683253bb 100644
--- a/spec/models/project_services/bamboo_service_spec.rb
+++ b/spec/models/project_services/bamboo_service_spec.rb
@@ -1,6 +1,6 @@
require 'spec_helper'
-describe BambooService, models: true, caching: true do
+describe BambooService, :use_clean_rails_memory_store_caching, models: true do
include ReactiveCachingHelpers
let(:bamboo_url) { 'http://gitlab.com/bamboo' }
diff --git a/spec/models/project_services/buildkite_service_spec.rb b/spec/models/project_services/buildkite_service_spec.rb
index dd529597067be64f19633ed903452b2da5b31674..b4ee6691e6704de84c8056358ab19e04c21e14c5 100644
--- a/spec/models/project_services/buildkite_service_spec.rb
+++ b/spec/models/project_services/buildkite_service_spec.rb
@@ -1,6 +1,6 @@
require 'spec_helper'
-describe BuildkiteService, models: true, caching: true do
+describe BuildkiteService, :use_clean_rails_memory_store_caching, models: true do
include ReactiveCachingHelpers
let(:project) { create(:empty_project) }
diff --git a/spec/models/project_services/drone_ci_service_spec.rb b/spec/models/project_services/drone_ci_service_spec.rb
index 1400175427f96948858992d3d0c00777afd15280..c9ac256ff383fcfa0848d05534cc483304ab04af 100644
--- a/spec/models/project_services/drone_ci_service_spec.rb
+++ b/spec/models/project_services/drone_ci_service_spec.rb
@@ -1,6 +1,6 @@
require 'spec_helper'
-describe DroneCiService, models: true, caching: true do
+describe DroneCiService, :use_clean_rails_memory_store_caching, models: true do
include ReactiveCachingHelpers
describe 'associations' do
diff --git a/spec/models/project_services/kubernetes_service_spec.rb b/spec/models/project_services/kubernetes_service_spec.rb
index 5ba523a478ad18026a2a84c0294cb177afe70043..b66bb5321ab0ff93f6ebcbc9d779182e41c4a381 100644
--- a/spec/models/project_services/kubernetes_service_spec.rb
+++ b/spec/models/project_services/kubernetes_service_spec.rb
@@ -1,6 +1,6 @@
require 'spec_helper'
-describe KubernetesService, models: true, caching: true do
+describe KubernetesService, :use_clean_rails_memory_store_caching, models: true do
include KubernetesHelpers
include ReactiveCachingHelpers
diff --git a/spec/models/project_services/prometheus_service_spec.rb b/spec/models/project_services/prometheus_service_spec.rb
index 37f23b1243c41c5f14fe6a4615dc9b416bc9d1f2..3fb134ec3b7ff8d3057036ead01e02f4b55ff52a 100644
--- a/spec/models/project_services/prometheus_service_spec.rb
+++ b/spec/models/project_services/prometheus_service_spec.rb
@@ -1,6 +1,6 @@
require 'spec_helper'
-describe PrometheusService, models: true, caching: true do
+describe PrometheusService, :use_clean_rails_memory_store_caching, models: true do
include PrometheusHelpers
include ReactiveCachingHelpers
diff --git a/spec/models/project_services/teamcity_service_spec.rb b/spec/models/project_services/teamcity_service_spec.rb
index 6b0040985109239bbf84a3cf403a6032bb7e2264..3f3a74d0f961121e1fa9c17e9ad3263cc38157e3 100644
--- a/spec/models/project_services/teamcity_service_spec.rb
+++ b/spec/models/project_services/teamcity_service_spec.rb
@@ -1,6 +1,6 @@
require 'spec_helper'
-describe TeamcityService, models: true, caching: true do
+describe TeamcityService, :use_clean_rails_memory_store_caching, models: true do
include ReactiveCachingHelpers
let(:teamcity_url) { 'http://gitlab.com/teamcity' }
diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb
index 99bfab7008887d1077bea0ff7e734cc7ff59dafa..c4bc129dd37e3069a78a8e0c4315a33c33e701ac 100644
--- a/spec/models/project_spec.rb
+++ b/spec/models/project_spec.rb
@@ -899,7 +899,7 @@ def create_pipeline
end
end
- describe '.cached_count', caching: true do
+ describe '.cached_count', :use_clean_rails_memory_store_caching do
let(:group) { create(:group, :public) }
let!(:project1) { create(:empty_project, :public, group: group) }
let!(:project2) { create(:empty_project, :public, group: group) }
diff --git a/spec/models/repository_spec.rb b/spec/models/repository_spec.rb
index af305e9b23473348e242f889d19b117dfdd4db31..7635b0868e7b4aaaf0dfd1d429891b57247e99b1 100644
--- a/spec/models/repository_spec.rb
+++ b/spec/models/repository_spec.rb
@@ -561,7 +561,7 @@
end
end
- describe "#changelog", caching: true do
+ describe "#changelog", :use_clean_rails_memory_store_caching do
it 'accepts changelog' do
expect(repository.tree).to receive(:blobs).and_return([TestBlob.new('changelog')])
@@ -593,7 +593,7 @@
end
end
- describe "#license_blob", caching: true do
+ describe "#license_blob", :use_clean_rails_memory_store_caching do
before do
repository.delete_file(
user, 'LICENSE', message: 'Remove LICENSE', branch_name: 'master')
@@ -638,7 +638,7 @@
end
end
- describe '#license_key', caching: true do
+ describe '#license_key', :use_clean_rails_memory_store_caching do
before do
repository.delete_file(user, 'LICENSE',
message: 'Remove LICENSE', branch_name: 'master')
@@ -703,7 +703,7 @@
end
end
- describe "#gitlab_ci_yml", caching: true do
+ describe "#gitlab_ci_yml", :use_clean_rails_memory_store_caching do
it 'returns valid file' do
files = [TestBlob.new('file'), TestBlob.new('.gitlab-ci.yml'), TestBlob.new('copying')]
expect(repository.tree).to receive(:blobs).and_return(files)
@@ -1611,7 +1611,7 @@ def merge(repository, user, merge_request, options = {})
end
end
- describe '#contribution_guide', caching: true do
+ describe '#contribution_guide', :use_clean_rails_memory_store_caching do
it 'returns and caches the output' do
expect(repository).to receive(:file_on_head)
.with(:contributing)
@@ -1625,7 +1625,7 @@ def merge(repository, user, merge_request, options = {})
end
end
- describe '#gitignore', caching: true do
+ describe '#gitignore', :use_clean_rails_memory_store_caching do
it 'returns and caches the output' do
expect(repository).to receive(:file_on_head)
.with(:gitignore)
@@ -1638,7 +1638,7 @@ def merge(repository, user, merge_request, options = {})
end
end
- describe '#koding_yml', caching: true do
+ describe '#koding_yml', :use_clean_rails_memory_store_caching do
it 'returns and caches the output' do
expect(repository).to receive(:file_on_head)
.with(:koding)
@@ -1651,7 +1651,7 @@ def merge(repository, user, merge_request, options = {})
end
end
- describe '#readme', caching: true do
+ describe '#readme', :use_clean_rails_memory_store_caching do
context 'with a non-existing repository' do
it 'returns nil' do
allow(repository).to receive(:tree).with(:head).and_return(nil)
@@ -1822,7 +1822,7 @@ def merge(repository, user, merge_request, options = {})
end
end
- describe '#cache_method_output', caching: true do
+ describe '#cache_method_output', :use_clean_rails_memory_store_caching do
context 'with a non-existing repository' do
let(:value) do
repository.cache_method_output(:cats, fallback: 10) do
diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb
index 448555d2190a0b67cfe32c8abf2e10141fa8f5ea..d04162a527f79e0f86961c40c6908db9f590fb27 100644
--- a/spec/models/user_spec.rb
+++ b/spec/models/user_spec.rb
@@ -348,7 +348,7 @@
end
end
- describe '#update_tracked_fields!', :redis do
+ describe '#update_tracked_fields!', :clean_gitlab_redis_shared_state do
let(:request) { OpenStruct.new(remote_ip: "127.0.0.1") }
let(:user) { create(:user) }
@@ -1684,7 +1684,7 @@ def add_user(access)
end
end
- describe '#refresh_authorized_projects', redis: true do
+ describe '#refresh_authorized_projects', clean_gitlab_redis_shared_state: true do
let(:project1) { create(:empty_project) }
let(:project2) { create(:empty_project) }
let(:user) { create(:user) }
diff --git a/spec/requests/api/internal_spec.rb b/spec/requests/api/internal_spec.rb
index cde4fa888a0f2d364bfeb2d8240b0b9edb97f54f..453eb4683a04f7dbcf374575a2cca720dbe97246 100644
--- a/spec/requests/api/internal_spec.rb
+++ b/spec/requests/api/internal_spec.rb
@@ -168,7 +168,7 @@
end
end
- describe "POST /internal/allowed", :redis do
+ describe "POST /internal/allowed", :clean_gitlab_redis_shared_state do
context "access granted" do
before do
project.team << [user, :developer]
diff --git a/spec/requests/api/users_spec.rb b/spec/requests/api/users_spec.rb
index c34b88f07410b570c98fcc32b466042272351055..a2368c9d996dff20cde4416bee3bd9069ab1e9d2 100644
--- a/spec/requests/api/users_spec.rb
+++ b/spec/requests/api/users_spec.rb
@@ -1313,7 +1313,7 @@
end
end
- context "user activities", :redis do
+ context "user activities", :clean_gitlab_redis_shared_state do
let!(:old_active_user) { create(:user, last_activity_on: Time.utc(2000, 1, 1)) }
let!(:newly_active_user) { create(:user, last_activity_on: 2.days.ago.midday) }
diff --git a/spec/requests/git_http_spec.rb b/spec/requests/git_http_spec.rb
index 185679e1a0fbd8b4563259768b8073f20e100f0e..d0443a450a2a2d8a837fa9b215896171d133c83c 100644
--- a/spec/requests/git_http_spec.rb
+++ b/spec/requests/git_http_spec.rb
@@ -406,7 +406,7 @@
end
end
- it 'updates the user last activity', :redis do
+ it 'updates the user last activity', :clean_gitlab_redis_shared_state do
expect(user_activity(user)).to be_nil
download(path, env) do |response|
diff --git a/spec/requests/openid_connect_spec.rb b/spec/requests/openid_connect_spec.rb
index 6d1f0b24196b6f30bf74ac67d52a3e804eec1ffc..ebba28ba8cea9b0bca60e5eb1f19fc53ac05fe19 100644
--- a/spec/requests/openid_connect_spec.rb
+++ b/spec/requests/openid_connect_spec.rb
@@ -98,7 +98,7 @@ def hashed_subject
expect(@payload['sub']).to eq hashed_subject
end
- it 'includes the time of the last authentication', :redis do
+ it 'includes the time of the last authentication', :clean_gitlab_redis_shared_state do
expect(@payload['auth_time']).to eq user.current_sign_in_at.to_i
end
diff --git a/spec/services/event_create_service_spec.rb b/spec/services/event_create_service_spec.rb
index b06cefe071d84a0014afeabfbc1e36ef0528c202..8d067c194cc0d0e210c57b8fabd1224a3920565e 100644
--- a/spec/services/event_create_service_spec.rb
+++ b/spec/services/event_create_service_spec.rb
@@ -113,7 +113,7 @@
end
end
- describe '#push', :redis do
+ describe '#push', :clean_gitlab_redis_shared_state do
let(:project) { create(:empty_project) }
let(:user) { create(:user) }
diff --git a/spec/services/git_push_service_spec.rb b/spec/services/git_push_service_spec.rb
index 8e8816870e1a30919233738eac30bfd5dbde6538..3f77ed1006990ba09750f8e89ff862fd6f07a191 100644
--- a/spec/services/git_push_service_spec.rb
+++ b/spec/services/git_push_service_spec.rb
@@ -527,14 +527,18 @@
let(:housekeeping) { Projects::HousekeepingService.new(project) }
before do
- # Flush any raw Redis data stored by the housekeeping code.
- Gitlab::Redis.with { |conn| conn.flushall }
+ # Flush any raw key-value data stored by the housekeeping code.
+ Gitlab::Redis::Cache.with { |conn| conn.flushall }
+ Gitlab::Redis::Queues.with { |conn| conn.flushall }
+ Gitlab::Redis::SharedState.with { |conn| conn.flushall }
allow(Projects::HousekeepingService).to receive(:new).and_return(housekeeping)
end
after do
- Gitlab::Redis.with { |conn| conn.flushall }
+ Gitlab::Redis::Cache.with { |conn| conn.flushall }
+ Gitlab::Redis::Queues.with { |conn| conn.flushall }
+ Gitlab::Redis::SharedState.with { |conn| conn.flushall }
end
it 'does not perform housekeeping when not needed' do
diff --git a/spec/services/milestones/destroy_service_spec.rb b/spec/services/milestones/destroy_service_spec.rb
deleted file mode 100644
index 8d1fe3ae2c1e14a7115a3fea41f83c73432e3751..0000000000000000000000000000000000000000
--- a/spec/services/milestones/destroy_service_spec.rb
+++ /dev/null
@@ -1,51 +0,0 @@
-require 'spec_helper'
-
-describe Milestones::DestroyService, services: true do
- let(:user) { create(:user) }
- let(:project) { create(:project) }
- let(:milestone) { create(:milestone, title: 'Milestone v1.0', project: project) }
- let(:issue) { create(:issue, project: project, milestone: milestone) }
- let(:merge_request) { create(:merge_request, source_project: project, milestone: milestone) }
-
- before do
- project.team << [user, :master]
- end
-
- def service
- described_class.new(project, user, {})
- end
-
- describe '#execute' do
- it 'deletes milestone' do
- service.execute(milestone)
-
- expect { milestone.reload }.to raise_error ActiveRecord::RecordNotFound
- end
-
- it 'deletes milestone id from issuables' do
- service.execute(milestone)
-
- expect(issue.reload.milestone).to be_nil
- expect(merge_request.reload.milestone).to be_nil
- end
-
- context 'group milestones' do
- let(:group) { create(:group) }
- let(:group_milestone) { create(:milestone, group: group) }
-
- before do
- project.update(namespace: group)
- group.add_developer(user)
- end
-
- it { expect(service.execute(group_milestone)).to be_nil }
-
- it 'does not update milestone issuables' do
- expect(MergeRequests::UpdateService).not_to receive(:new)
- expect(Issues::UpdateService).not_to receive(:new)
-
- service.execute(group_milestone)
- end
- end
- end
-end
diff --git a/spec/services/projects/destroy_service_spec.rb b/spec/services/projects/destroy_service_spec.rb
index 697dc18feb02f462af938d95d7fe87553c832fc5..b399d3402fd0b5e680b3be5567acf25582854217 100644
--- a/spec/services/projects/destroy_service_spec.rb
+++ b/spec/services/projects/destroy_service_spec.rb
@@ -60,14 +60,14 @@
before do
new_user = create(:user)
project.team.add_user(new_user, Gitlab::Access::DEVELOPER)
- allow_any_instance_of(Projects::DestroyService).to receive(:flush_caches).and_raise(Redis::CannotConnectError)
+ allow_any_instance_of(Projects::DestroyService).to receive(:flush_caches).and_raise(::Redis::CannotConnectError)
end
it 'keeps project team intact upon an error' do
Sidekiq::Testing.inline! do
begin
destroy_project(project, user, {})
- rescue Redis::CannotConnectError
+ rescue ::Redis::CannotConnectError
end
end
diff --git a/spec/services/todo_service_spec.rb b/spec/services/todo_service_spec.rb
index 175a42a32d9b707b53007989f09342a595bb82d0..de41cbab14cb3b3e5f29220ebe34e580c89ea015 100644
--- a/spec/services/todo_service_spec.rb
+++ b/spec/services/todo_service_spec.rb
@@ -908,7 +908,7 @@
end
end
- it 'caches the number of todos of a user', :caching do
+ it 'caches the number of todos of a user', :use_clean_rails_memory_store_caching do
create(:todo, :mentioned, user: john_doe, target: issue, project: project)
todo = create(:todo, :mentioned, user: john_doe, target: issue, project: project)
TodoService.new.mark_todos_as_done([todo], john_doe)
diff --git a/spec/services/users/activity_service_spec.rb b/spec/services/users/activity_service_spec.rb
index 2e009d4ce1c28ea61a468c997a9c546c8aaa6547..e5330d1d3e45396fc55b744bca056cdbdcba5ae0 100644
--- a/spec/services/users/activity_service_spec.rb
+++ b/spec/services/users/activity_service_spec.rb
@@ -7,7 +7,7 @@
subject(:service) { described_class.new(user, 'type') }
- describe '#execute', :redis do
+ describe '#execute', :clean_gitlab_redis_shared_state do
context 'when last activity is nil' do
before do
service.execute
diff --git a/spec/services/users/refresh_authorized_projects_service_spec.rb b/spec/services/users/refresh_authorized_projects_service_spec.rb
index b65cadbb2f593bb111c0d3f0a283b60342e01e2b..1c0f55d29651c8a7f9861aa19f785ad0c94ac7c4 100644
--- a/spec/services/users/refresh_authorized_projects_service_spec.rb
+++ b/spec/services/users/refresh_authorized_projects_service_spec.rb
@@ -8,7 +8,7 @@
let(:user) { project.namespace.owner }
let(:service) { described_class.new(user) }
- describe '#execute', :redis do
+ describe '#execute', :clean_gitlab_redis_shared_state do
it 'refreshes the authorizations using a lease' do
expect_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain)
.and_return('foo')
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index a497b8613bbf2613e8b9b545fa66857654c20c81..dc0bdd9f4c72ba4354223fb3e29b5a7974a076e9 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -91,20 +91,30 @@
end
end
- config.around(:each, :caching) do |example|
+ config.around(:each, :use_clean_rails_memory_store_caching) do |example|
caching_store = Rails.cache
- Rails.cache = ActiveSupport::Cache::MemoryStore.new if example.metadata[:caching]
+ Rails.cache = ActiveSupport::Cache::MemoryStore.new
+
example.run
+
Rails.cache = caching_store
end
- config.around(:each, :redis) do |example|
- Gitlab::Redis.with(&:flushall)
+ config.around(:each, :clean_gitlab_redis_cache) do |example|
+ Gitlab::Redis::Cache.with(&:flushall)
+
+ example.run
+
+ Gitlab::Redis::Cache.with(&:flushall)
+ end
+
+ config.around(:each, :clean_gitlab_redis_shared_state) do |example|
+ Gitlab::Redis::SharedState.with(&:flushall)
Sidekiq.redis(&:flushall)
example.run
- Gitlab::Redis.with(&:flushall)
+ Gitlab::Redis::SharedState.with(&:flushall)
Sidekiq.redis(&:flushall)
end
diff --git a/spec/lib/gitlab/redis_spec.rb b/spec/support/redis/redis_shared_examples.rb
similarity index 69%
rename from spec/lib/gitlab/redis_spec.rb
rename to spec/support/redis/redis_shared_examples.rb
index 593aa5038ada36a159098f675fd8b0656333d304..95a5df181c18c67f4bb149261f94ef19864cd32f 100644
--- a/spec/lib/gitlab/redis_spec.rb
+++ b/spec/support/redis/redis_shared_examples.rb
@@ -1,12 +1,10 @@
-require 'spec_helper'
-
-describe Gitlab::Redis do
+RSpec.shared_examples "redis_shared_examples" do
include StubENV
- let(:config) { 'config/resque.yml' }
+ let(:test_redis_url) { "redis://redishost:#{redis_port}"}
before(:each) do
- stub_env('GITLAB_REDIS_CONFIG_FILE', Rails.root.join(config).to_s)
+ stub_env(environment_config_file_name, Rails.root.join(config_file_name))
clear_raw_config
end
@@ -26,46 +24,40 @@
end
context 'when url contains unix socket reference' do
- let(:config_old) { 'spec/fixtures/config/redis_old_format_socket.yml' }
- let(:config_new) { 'spec/fixtures/config/redis_new_format_socket.yml' }
-
context 'with old format' do
- let(:config) { config_old }
+ let(:config_file_name) { config_old_format_socket }
it 'returns path key instead' do
- is_expected.to include(path: '/path/to/old/redis.sock')
+ is_expected.to include(path: old_socket_path)
is_expected.not_to have_key(:url)
end
end
context 'with new format' do
- let(:config) { config_new }
+ let(:config_file_name) { config_new_format_socket }
it 'returns path key instead' do
- is_expected.to include(path: '/path/to/redis.sock')
+ is_expected.to include(path: new_socket_path)
is_expected.not_to have_key(:url)
end
end
end
context 'when url is host based' do
- let(:config_old) { 'spec/fixtures/config/redis_old_format_host.yml' }
- let(:config_new) { 'spec/fixtures/config/redis_new_format_host.yml' }
-
context 'with old format' do
- let(:config) { config_old }
+ let(:config_file_name) { config_old_format_host }
it 'returns hash with host, port, db, and password' do
- is_expected.to include(host: 'localhost', password: 'mypassword', port: 6379, db: 99)
+ is_expected.to include(host: 'localhost', password: 'mypassword', port: redis_port, db: redis_database)
is_expected.not_to have_key(:url)
end
end
context 'with new format' do
- let(:config) { config_new }
+ let(:config_file_name) { config_new_format_host }
it 'returns hash with host, port, db, and password' do
- is_expected.to include(host: 'localhost', password: 'mynewpassword', port: 6379, db: 99)
+ is_expected.to include(host: 'localhost', password: 'mynewpassword', port: redis_port, db: redis_database)
is_expected.not_to have_key(:url)
end
end
@@ -82,21 +74,21 @@
end
context 'when yml file with env variable' do
- let(:config) { 'spec/fixtures/config/redis_config_with_env.yml' }
+ let(:config_file_name) { config_with_environment_variable_inside }
before do
- stub_env('TEST_GITLAB_REDIS_URL', 'redis://redishost:6379')
+ stub_env(config_env_variable_url, test_redis_url)
end
it 'reads redis url from env variable' do
- expect(described_class.url).to eq 'redis://redishost:6379'
+ expect(described_class.url).to eq test_redis_url
end
end
end
describe '._raw_config' do
subject { described_class._raw_config }
- let(:config) { '/var/empty/doesnotexist' }
+ let(:config_file_name) { '/var/empty/doesnotexist' }
it 'should be frozen' do
expect(subject).to be_frozen
@@ -124,7 +116,7 @@
it 'instantiates a connection pool with size 5' do
expect(ConnectionPool).to receive(:new).with(size: 5).and_call_original
- described_class.with { |_redis| true }
+ described_class.with { |_redis_shared_example| true }
end
end
@@ -137,7 +129,7 @@
it 'instantiates a connection pool with a size based on the concurrency of the worker' do
expect(ConnectionPool).to receive(:new).with(size: 18 + 5).and_call_original
- described_class.with { |_redis| true }
+ described_class.with { |_redis_shared_example| true }
end
end
end
@@ -146,16 +138,16 @@
subject { described_class.new(Rails.env).sentinels }
context 'when sentinels are defined' do
- let(:config) { 'spec/fixtures/config/redis_new_format_host.yml' }
+ let(:config_file_name) { config_new_format_host }
it 'returns an array of hashes with host and port keys' do
- is_expected.to include(host: 'localhost', port: 26380)
- is_expected.to include(host: 'slave2', port: 26381)
+ is_expected.to include(host: 'localhost', port: sentinel_port)
+ is_expected.to include(host: 'slave2', port: sentinel_port)
end
end
context 'when sentinels are not defined' do
- let(:config) { 'spec/fixtures/config/redis_old_format_host.yml' }
+ let(:config_file_name) { config_old_format_host }
it 'returns nil' do
is_expected.to be_nil
@@ -167,7 +159,7 @@
subject { described_class.new(Rails.env).sentinels? }
context 'when sentinels are defined' do
- let(:config) { 'spec/fixtures/config/redis_new_format_host.yml' }
+ let(:config_file_name) { config_new_format_host }
it 'returns true' do
is_expected.to be_truthy
@@ -175,7 +167,7 @@
end
context 'when sentinels are not defined' do
- let(:config) { 'spec/fixtures/config/redis_old_format_host.yml' }
+ let(:config_file_name) { config_old_format_host }
it 'returns false' do
is_expected.to be_falsey
@@ -187,12 +179,12 @@
it 'returns default redis url when no config file is present' do
expect(subject).to receive(:fetch_config) { false }
- expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::Redis::DEFAULT_REDIS_URL)
+ expect(subject.send(:raw_config_hash)).to eq(url: class_redis_url )
end
it 'returns old-style single url config in a hash' do
- expect(subject).to receive(:fetch_config) { 'redis://myredis:6379' }
- expect(subject.send(:raw_config_hash)).to eq(url: 'redis://myredis:6379')
+ expect(subject).to receive(:fetch_config) { test_redis_url }
+ expect(subject.send(:raw_config_hash)).to eq(url: test_redis_url)
end
end
diff --git a/spec/support/unique_ip_check_shared_examples.rb b/spec/support/unique_ip_check_shared_examples.rb
index 1986d202c4ac17b59af94b6c5799d3c8208fbab5..ff0b47899f5308b4a1194d1bdc4f904f2f7ea525 100644
--- a/spec/support/unique_ip_check_shared_examples.rb
+++ b/spec/support/unique_ip_check_shared_examples.rb
@@ -1,7 +1,9 @@
shared_context 'unique ips sign in limit' do
include StubENV
before(:each) do
- Gitlab::Redis.with(&:flushall)
+ Gitlab::Redis::Cache.with(&:flushall)
+ Gitlab::Redis::Queues.with(&:flushall)
+ Gitlab::Redis::SharedState.with(&:flushall)
end
before do
diff --git a/spec/workers/schedule_update_user_activity_worker_spec.rb b/spec/workers/schedule_update_user_activity_worker_spec.rb
index e583c3203aa04a48f1c63bbadb7f54e0bead453e..32c59381b01f34f6c329be6639eee60b1c5df70d 100644
--- a/spec/workers/schedule_update_user_activity_worker_spec.rb
+++ b/spec/workers/schedule_update_user_activity_worker_spec.rb
@@ -1,6 +1,6 @@
require 'spec_helper'
-describe ScheduleUpdateUserActivityWorker, :redis do
+describe ScheduleUpdateUserActivityWorker, :clean_gitlab_redis_shared_state do
let(:now) { Time.now }
before do
diff --git a/spec/workers/update_user_activity_worker_spec.rb b/spec/workers/update_user_activity_worker_spec.rb
index 43e9511f11654ae88e7ad6c2c4eb7116830df5f1..268ca1d81f280d097038dae545c3869abe5c2b23 100644
--- a/spec/workers/update_user_activity_worker_spec.rb
+++ b/spec/workers/update_user_activity_worker_spec.rb
@@ -1,6 +1,6 @@
require 'spec_helper'
-describe UpdateUserActivityWorker, :redis do
+describe UpdateUserActivityWorker, :clean_gitlab_redis_shared_state do
let(:user_active_2_days_ago) { create(:user, current_sign_in_at: 10.months.ago) }
let(:user_active_yesterday_1) { create(:user) }
let(:user_active_yesterday_2) { create(:user) }
@@ -25,7 +25,7 @@
end
end
- it 'deletes the pairs from Redis' do
+ it 'deletes the pairs from SharedState' do
data.each { |id, time| Gitlab::UserActivities.record(id, time) }
subject.perform(data)