diff --git a/app/models/application_record.rb b/app/models/application_record.rb index 7737614ae251b536f14333649f31f5c3213cfd50..0f7b6388441e8df06a21ae0f0d8d14e444ff9c8b 100644 --- a/app/models/application_record.rb +++ b/app/models/application_record.rb @@ -1,6 +1,10 @@ # frozen_string_literal: true class ApplicationRecord < ActiveRecord::Base + include DatabaseReflection + include Transactions + include LegacyBulkInsert + self.abstract_class = true alias_method :reset, :reload diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb index 028279b6150c1daed9eebd21212e1fdf8ef99d04..54ec8b2c3e429c34d2215427812652729ad6a231 100644 --- a/app/models/application_setting_implementation.rb +++ b/app/models/application_setting_implementation.rb @@ -244,11 +244,11 @@ def human_attribute_name(attr, _options = {}) end def home_page_url_column_exists? - ::Gitlab::Database.main.cached_column_exists?(:application_settings, :home_page_url) + ApplicationSetting.database.cached_column_exists?(:home_page_url) end def help_page_support_url_column_exists? - ::Gitlab::Database.main.cached_column_exists?(:application_settings, :help_page_support_url) + ApplicationSetting.database.cached_column_exists?(:help_page_support_url) end def disabled_oauth_sign_in_sources=(sources) diff --git a/app/models/concerns/cascading_namespace_setting_attribute.rb b/app/models/concerns/cascading_namespace_setting_attribute.rb index c70c6dca1058171c606354f58d21a7f7d091583d..731729a1ed528321d9747e2376613b794fd0ccf4 100644 --- a/app/models/concerns/cascading_namespace_setting_attribute.rb +++ b/app/models/concerns/cascading_namespace_setting_attribute.rb @@ -127,7 +127,7 @@ def define_lock_methods(attribute) end def alias_boolean(attribute) - return unless Gitlab::Database.main.exists? && type_for_attribute(attribute).type == :boolean + return unless database.exists? && type_for_attribute(attribute).type == :boolean alias_method :"#{attribute}?", attribute end diff --git a/app/models/concerns/database_reflection.rb b/app/models/concerns/database_reflection.rb new file mode 100644 index 0000000000000000000000000000000000000000..1842f5bf4ecd76cea4b6def02edc889900fd4028 --- /dev/null +++ b/app/models/concerns/database_reflection.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +# A module that makes it easier/less verbose to reflect upon a database +# connection. +# +# Using this module you can write this: +# +# User.database.database_name +# +# Instead of this: +# +# Gitlab::Database::Reflection.new(User).database_name +module DatabaseReflection + extend ActiveSupport::Concern + + class_methods do + def database + @database_reflection ||= ::Gitlab::Database::Reflection.new(self) + end + end +end diff --git a/app/models/concerns/legacy_bulk_insert.rb b/app/models/concerns/legacy_bulk_insert.rb new file mode 100644 index 0000000000000000000000000000000000000000..1249dfb70cd64a3af3a46ef8c06e5f53684515fc --- /dev/null +++ b/app/models/concerns/legacy_bulk_insert.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module LegacyBulkInsert + extend ActiveSupport::Concern + + class_methods do + # Bulk inserts a number of rows into a table, optionally returning their + # IDs. + # + # This method is deprecated, and you should use the BulkInsertSafe module + # instead. + # + # table - The name of the table to insert the rows into. + # rows - An Array of Hash instances, each mapping the columns to their + # values. + # return_ids - When set to true the return value will be an Array of IDs of + # the inserted rows + # disable_quote - A key or an Array of keys to exclude from quoting (You + # become responsible for protection from SQL injection for + # these keys!) + # on_conflict - Defines an upsert. Values can be: :disabled (default) or + # :do_nothing + def legacy_bulk_insert(table, rows, return_ids: false, disable_quote: [], on_conflict: nil) + return if rows.empty? + + keys = rows.first.keys + columns = keys.map { |key| connection.quote_column_name(key) } + + disable_quote = Array(disable_quote).to_set + tuples = rows.map do |row| + keys.map do |k| + disable_quote.include?(k) ? row[k] : connection.quote(row[k]) + end + end + + sql = <<-EOF + INSERT INTO #{table} (#{columns.join(', ')}) + VALUES #{tuples.map { |tuple| "(#{tuple.join(', ')})" }.join(', ')} + EOF + + sql = "#{sql} ON CONFLICT DO NOTHING" if on_conflict == :do_nothing + + sql = "#{sql} RETURNING id" if return_ids + + result = connection.execute(sql) + + if return_ids + result.values.map { |tuple| tuple[0].to_i } + else + [] + end + end + end +end diff --git a/app/models/concerns/sha256_attribute.rb b/app/models/concerns/sha256_attribute.rb index 17fda6c806c3bc5fbfec44015869fa86e1a37b72..3c906642b1a398a1d6f77b66e076c812ba2a22f7 100644 --- a/app/models/concerns/sha256_attribute.rb +++ b/app/models/concerns/sha256_attribute.rb @@ -39,7 +39,7 @@ def validate_binary_column_exists!(name) end def database_exists? - Gitlab::Database.main.exists? + database.exists? end end end diff --git a/app/models/concerns/sha_attribute.rb b/app/models/concerns/sha_attribute.rb index 27277bc52963c8fab8153a7869ad740bfd0baf9d..ba7c6c0cd8b2ddbd0f9edfaf4b57b851a4b8fbaa 100644 --- a/app/models/concerns/sha_attribute.rb +++ b/app/models/concerns/sha_attribute.rb @@ -32,7 +32,7 @@ def validate_binary_column_exists!(name) end def database_exists? - Gitlab::Database.main.exists? + database.exists? end end end diff --git a/app/models/concerns/transactions.rb b/app/models/concerns/transactions.rb new file mode 100644 index 0000000000000000000000000000000000000000..a186ebc847570e31f815eb8710641442727f45fa --- /dev/null +++ b/app/models/concerns/transactions.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Transactions + extend ActiveSupport::Concern + + class_methods do + # inside_transaction? will return true if the caller is running within a + # transaction. Handles special cases when running inside a test environment, + # where tests may be wrapped in transactions + def inside_transaction? + base = Rails.env.test? ? @open_transactions_baseline.to_i : 0 + + connection.open_transactions > base + end + + # These methods that access @open_transactions_baseline are not thread-safe. + # These are fine though because we only call these in RSpec's main thread. + # If we decide to run specs multi-threaded, we would need to use something + # like ThreadGroup to keep track of this value + def set_open_transactions_baseline + @open_transactions_baseline = connection.open_transactions + end + + def reset_open_transactions_baseline + @open_transactions_baseline = 0 + end + end +end diff --git a/app/models/concerns/x509_serial_number_attribute.rb b/app/models/concerns/x509_serial_number_attribute.rb index dfb1e151b4117a608967dd881da8dc4a63dfb7a4..e51ed95bf7051dbf6b09e8878618d204aec5e75c 100644 --- a/app/models/concerns/x509_serial_number_attribute.rb +++ b/app/models/concerns/x509_serial_number_attribute.rb @@ -39,7 +39,7 @@ def validate_binary_column_exists!(name) end def database_exists? - Gitlab::Database.main.exists? + database.exists? end end end diff --git a/app/models/deployment.rb b/app/models/deployment.rb index f7a7f7a4a5f20c37e143ee7e935e81990a0a74b1..5fddc6616029acc3630e7d114dbb71acd42f4825 100644 --- a/app/models/deployment.rb +++ b/app/models/deployment.rb @@ -300,7 +300,7 @@ def link_merge_requests(relation) "#{id} as deployment_id", "#{environment_id} as environment_id").to_sql - # We don't use `Gitlab::Database.main.bulk_insert` here so that we don't need to + # We don't use `ApplicationRecord.legacy_bulk_insert` here so that we don't need to # first pluck lots of IDs into memory. # # We also ignore any duplicates so this method can be called multiple times diff --git a/app/models/design_management/version.rb b/app/models/design_management/version.rb index 6cda03557d1ff1c2d7e29167e824758f81e85557..5819404efb9f9996d19b58c6e0626d45c6e27f33 100644 --- a/app/models/design_management/version.rb +++ b/app/models/design_management/version.rb @@ -88,7 +88,7 @@ def self.create_for_designs(design_actions, sha, author) rows = design_actions.map { |action| action.row_attrs(version) } - Gitlab::Database.main.bulk_insert(::DesignManagement::Action.table_name, rows) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert(::DesignManagement::Action.table_name, rows) # rubocop:disable Gitlab/BulkInsert version.designs.reset version.validate! design_actions.each(&:performed) diff --git a/app/models/merge_request_context_commit.rb b/app/models/merge_request_context_commit.rb index 09824ed44680264464bb53e21a3443319840e51a..ebbdecf8aa75d91aabbcf7303f6cb781bc2ae08f 100644 --- a/app/models/merge_request_context_commit.rb +++ b/app/models/merge_request_context_commit.rb @@ -26,7 +26,7 @@ def self.delete_bulk(merge_request, commits) # create MergeRequestContextCommit by given commit sha and it's diff file record def self.bulk_insert(rows, **args) - Gitlab::Database.main.bulk_insert('merge_request_context_commits', rows, **args) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert('merge_request_context_commits', rows, **args) # rubocop:disable Gitlab/BulkInsert end def to_commit diff --git a/app/models/merge_request_context_commit_diff_file.rb b/app/models/merge_request_context_commit_diff_file.rb index b9efebe3af2bdd8655863f59d3212283bc1ce5f5..fdf570689287b4ecede8f8899f53ed49e55e11ef 100644 --- a/app/models/merge_request_context_commit_diff_file.rb +++ b/app/models/merge_request_context_commit_diff_file.rb @@ -14,7 +14,7 @@ class MergeRequestContextCommitDiffFile < ApplicationRecord # create MergeRequestContextCommitDiffFile by given diff file record(s) def self.bulk_insert(*args) - Gitlab::Database.main.bulk_insert('merge_request_context_commit_diff_files', *args) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert('merge_request_context_commit_diff_files', *args) # rubocop:disable Gitlab/BulkInsert end def path diff --git a/app/models/merge_request_diff.rb b/app/models/merge_request_diff.rb index 13ef47bec45b4dcb9c8c4342064dddbf14277b42..2516ff05bdac92423df95c5ff84b4b37bb3ace47 100644 --- a/app/models/merge_request_diff.rb +++ b/app/models/merge_request_diff.rb @@ -515,7 +515,7 @@ def migrate_files_to_external_storage! transaction do MergeRequestDiffFile.where(merge_request_diff_id: id).delete_all - Gitlab::Database.main.bulk_insert('merge_request_diff_files', rows) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert('merge_request_diff_files', rows) # rubocop:disable Gitlab/BulkInsert save! end @@ -535,7 +535,7 @@ def migrate_files_to_database! transaction do MergeRequestDiffFile.where(merge_request_diff_id: id).delete_all - Gitlab::Database.main.bulk_insert('merge_request_diff_files', rows) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert('merge_request_diff_files', rows) # rubocop:disable Gitlab/BulkInsert update!(stored_externally: false) end @@ -595,7 +595,7 @@ def create_merge_request_diff_files(rows) rows = build_external_merge_request_diff_files(rows) if use_external_diff? # Faster inserts - Gitlab::Database.main.bulk_insert('merge_request_diff_files', rows) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert('merge_request_diff_files', rows) # rubocop:disable Gitlab/BulkInsert end def build_external_diff_tempfile(rows) diff --git a/app/models/merge_request_diff_commit.rb b/app/models/merge_request_diff_commit.rb index b1cae0d1e4940a704812db518e4d74f8706b525d..66f1e45fd49365dabd8909bf222b6740815d7167 100644 --- a/app/models/merge_request_diff_commit.rb +++ b/app/models/merge_request_diff_commit.rb @@ -74,7 +74,7 @@ def self.create_bulk(merge_request_diff_id, commits) ) end - Gitlab::Database.main.bulk_insert(self.table_name, rows) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert(self.table_name, rows) # rubocop:disable Gitlab/BulkInsert end def self.prepare_commits_for_bulk_insert(commits) diff --git a/app/services/design_management/copy_design_collection/copy_service.rb b/app/services/design_management/copy_design_collection/copy_service.rb index c43696442d25ec6aac4c9b0b3db83f4ca158ead3..5e557e9ea53115f75ad9c905e258720718d028d4 100644 --- a/app/services/design_management/copy_design_collection/copy_service.rb +++ b/app/services/design_management/copy_design_collection/copy_service.rb @@ -181,12 +181,12 @@ def copy_designs! ) end - # TODO Replace `Gitlab::Database.main.bulk_insert` with `BulkInsertSafe` + # TODO Replace `ApplicationRecord.legacy_bulk_insert` with `BulkInsertSafe` # once https://gitlab.com/gitlab-org/gitlab/-/issues/247718 is fixed. # When this is fixed, we can remove the call to # `with_project_iid_supply` above, since the objects will be instantiated # and callbacks (including `ensure_project_iid!`) will fire. - ::Gitlab::Database.main.bulk_insert( # rubocop:disable Gitlab/BulkInsert + ::ApplicationRecord.legacy_bulk_insert( # rubocop:disable Gitlab/BulkInsert DesignManagement::Design.table_name, new_rows, return_ids: true @@ -207,9 +207,9 @@ def copy_versions! ) end - # TODO Replace `Gitlab::Database.main.bulk_insert` with `BulkInsertSafe` + # TODO Replace `ApplicationRecord.legacy_bulk_insert` with `BulkInsertSafe` # once https://gitlab.com/gitlab-org/gitlab/-/issues/247718 is fixed. - ::Gitlab::Database.main.bulk_insert( # rubocop:disable Gitlab/BulkInsert + ::ApplicationRecord.legacy_bulk_insert( # rubocop:disable Gitlab/BulkInsert DesignManagement::Version.table_name, new_rows, return_ids: true @@ -239,7 +239,7 @@ def copy_actions!(new_design_ids, new_version_ids) end # We cannot use `BulkInsertSafe` because of the uploader mounted in `Action`. - ::Gitlab::Database.main.bulk_insert( # rubocop:disable Gitlab/BulkInsert + ::ApplicationRecord.legacy_bulk_insert( # rubocop:disable Gitlab/BulkInsert DesignManagement::Action.table_name, new_rows ) @@ -278,7 +278,7 @@ def link_lfs_files! # We cannot use `BulkInsertSafe` due to the LfsObjectsProject#update_project_statistics # callback that fires after_commit. - ::Gitlab::Database.main.bulk_insert( # rubocop:disable Gitlab/BulkInsert + ::ApplicationRecord.legacy_bulk_insert( # rubocop:disable Gitlab/BulkInsert LfsObjectsProject.table_name, new_rows, on_conflict: :do_nothing # Upsert diff --git a/app/services/issuable/clone/attributes_rewriter.rb b/app/services/issuable/clone/attributes_rewriter.rb index d8b639bb422437f56efb11d39b23381cfaf4bd6f..279d30518486eeb812f8ee20ab0fa2af24e1f47f 100644 --- a/app/services/issuable/clone/attributes_rewriter.rb +++ b/app/services/issuable/clone/attributes_rewriter.rb @@ -99,7 +99,7 @@ def copy_events(table_name, events_to_copy) yield(event) end.compact - Gitlab::Database.main.bulk_insert(table_name, events) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert(table_name, events) # rubocop:disable Gitlab/BulkInsert end end diff --git a/app/services/packages/create_dependency_service.rb b/app/services/packages/create_dependency_service.rb index 2c80ec66dbc77619a8b230b0d9cfbf59ae70d302..10a86e44cb0a35b3f76bbc7387ebc72a5315e580 100644 --- a/app/services/packages/create_dependency_service.rb +++ b/app/services/packages/create_dependency_service.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true module Packages + # rubocop: disable Gitlab/BulkInsert class CreateDependencyService < BaseService attr_reader :package, :dependencies @@ -51,7 +52,7 @@ def bulk_insert_package_dependencies(names_and_version_patterns) } end - ids = database.bulk_insert(Packages::Dependency.table_name, rows, return_ids: true, on_conflict: :do_nothing) + ids = ApplicationRecord.legacy_bulk_insert(Packages::Dependency.table_name, rows, return_ids: true, on_conflict: :do_nothing) return ids if ids.size == names_and_version_patterns.size Packages::Dependency.uncached do @@ -72,11 +73,8 @@ def bulk_insert_package_dependency_links(type, dependency_ids) } end - database.bulk_insert(Packages::DependencyLink.table_name, rows) - end - - def database - ::Gitlab::Database.main + ApplicationRecord.legacy_bulk_insert(Packages::DependencyLink.table_name, rows) end end + # rubocop: enable Gitlab/BulkInsert end diff --git a/app/services/packages/nuget/create_dependency_service.rb b/app/services/packages/nuget/create_dependency_service.rb index 3fc42056d435ba8940d4da4cc205c7f7e84cef21..85f295ac7b7163a1f1a05b519ac5cb5cdb36f1e4 100644 --- a/app/services/packages/nuget/create_dependency_service.rb +++ b/app/services/packages/nuget/create_dependency_service.rb @@ -41,7 +41,7 @@ def create_dependency_link_metadata } end - ::Gitlab::Database.main.bulk_insert(::Packages::Nuget::DependencyLinkMetadatum.table_name, rows.compact) # rubocop:disable Gitlab/BulkInsert + ::ApplicationRecord.legacy_bulk_insert(::Packages::Nuget::DependencyLinkMetadatum.table_name, rows.compact) # rubocop:disable Gitlab/BulkInsert end def raw_dependency_for(dependency) diff --git a/app/services/packages/update_tags_service.rb b/app/services/packages/update_tags_service.rb index 2bdf75a6617056d33934f9a1d5c706e0caca117c..f29c54dacb94300c1642b2068a16b36faf7cc25e 100644 --- a/app/services/packages/update_tags_service.rb +++ b/app/services/packages/update_tags_service.rb @@ -15,7 +15,7 @@ def execute tags_to_create = @tags - existing_tags @package.tags.with_name(tags_to_destroy).delete_all if tags_to_destroy.any? - ::Gitlab::Database.main.bulk_insert(Packages::Tag.table_name, rows(tags_to_create)) if tags_to_create.any? # rubocop:disable Gitlab/BulkInsert + ::ApplicationRecord.legacy_bulk_insert(Packages::Tag.table_name, rows(tags_to_create)) if tags_to_create.any? # rubocop:disable Gitlab/BulkInsert end private diff --git a/app/services/projects/detect_repository_languages_service.rb b/app/services/projects/detect_repository_languages_service.rb index 0356a6b0ccd672363ae0272cec10ccd184a7bc13..9db0b71d106e525efdd98c630b755feb6f3b1b6a 100644 --- a/app/services/projects/detect_repository_languages_service.rb +++ b/app/services/projects/detect_repository_languages_service.rb @@ -21,7 +21,7 @@ def execute .update_all(share: update[:share]) end - Gitlab::Database.main.bulk_insert( # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert( # rubocop:disable Gitlab/BulkInsert RepositoryLanguage.table_name, detection.insertions(matching_programming_languages) ) diff --git a/app/services/projects/lfs_pointers/lfs_link_service.rb b/app/services/projects/lfs_pointers/lfs_link_service.rb index 7c00b9e6105c8c5bcec89f36056c2e30dc910a54..cf3cc5cd8e0b8629069408cc60629a47c24931a4 100644 --- a/app/services/projects/lfs_pointers/lfs_link_service.rb +++ b/app/services/projects/lfs_pointers/lfs_link_service.rb @@ -38,7 +38,7 @@ def link_existing_lfs_objects(oids) rows = existent_lfs_objects .not_linked_to_project(project) .map { |existing_lfs_object| { project_id: project.id, lfs_object_id: existing_lfs_object.id } } - Gitlab::Database.main.bulk_insert(:lfs_objects_projects, rows) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert(:lfs_objects_projects, rows) # rubocop:disable Gitlab/BulkInsert iterations += 1 linked_existing_objects += existent_lfs_objects.map(&:oid) diff --git a/app/services/resource_events/change_labels_service.rb b/app/services/resource_events/change_labels_service.rb index bc2d3a946ccad4e3ed214cf968a6b76a24d9c268..03ac839c509512757aa7b510fb00b7e898f810fc 100644 --- a/app/services/resource_events/change_labels_service.rb +++ b/app/services/resource_events/change_labels_service.rb @@ -23,7 +23,7 @@ def execute(added_labels: [], removed_labels: []) label_hash.merge(label_id: label.id, action: ResourceLabelEvent.actions['remove']) end - Gitlab::Database.main.bulk_insert(ResourceLabelEvent.table_name, labels) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert(ResourceLabelEvent.table_name, labels) # rubocop:disable Gitlab/BulkInsert resource.expire_note_etag_cache Gitlab::UsageDataCounters::IssueActivityUniqueCounter.track_issue_label_changed_action(author: user) if resource.is_a?(Issue) diff --git a/app/services/suggestions/create_service.rb b/app/services/suggestions/create_service.rb index eb98ed57d55d86dd27e7daf52b3416e11909d1f4..239cd86e0ece2f1b4e0b1f96b90cae62e7759dc9 100644 --- a/app/services/suggestions/create_service.rb +++ b/app/services/suggestions/create_service.rb @@ -25,7 +25,7 @@ def execute end rows.in_groups_of(100, false) do |rows| - Gitlab::Database.main.bulk_insert('suggestions', rows) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert('suggestions', rows) # rubocop:disable Gitlab/BulkInsert end Gitlab::UsageDataCounters::MergeRequestActivityUniqueCounter.track_add_suggestion_action(note: @note) diff --git a/app/views/admin/dashboard/index.html.haml b/app/views/admin/dashboard/index.html.haml index 681e7ccb61345de6263e08e389ad3eafb17b6730..4197d5b961f9fd6cc2197d36a7211242bc799f4d 100644 --- a/app/views/admin/dashboard/index.html.haml +++ b/app/views/admin/dashboard/index.html.haml @@ -153,9 +153,9 @@ %span.float-right #{Rails::VERSION::STRING} %p - = Gitlab::Database.main.human_adapter_name + = ApplicationRecord.database.human_adapter_name %span.float-right - = Gitlab::Database.main.version + = ApplicationRecord.database.version %p = _('Redis') %span.float-right diff --git a/app/workers/gitlab/jira_import/import_issue_worker.rb b/app/workers/gitlab/jira_import/import_issue_worker.rb index eabe7328b92e0c93c3d7313525adc6806b0c9e24..3824cc1f3efc163d9d0abe8fabc10195aec7cb27 100644 --- a/app/workers/gitlab/jira_import/import_issue_worker.rb +++ b/app/workers/gitlab/jira_import/import_issue_worker.rb @@ -54,7 +54,7 @@ def label_issue(project_id, issue_id, label_ids) label_link_attrs << build_label_attrs(issue_id, import_label_id.to_i) - Gitlab::Database.main.bulk_insert(LabelLink.table_name, label_link_attrs) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert(LabelLink.table_name, label_link_attrs) # rubocop:disable Gitlab/BulkInsert end def assign_issue(project_id, issue_id, assignee_ids) @@ -62,7 +62,7 @@ def assign_issue(project_id, issue_id, assignee_ids) assignee_attrs = assignee_ids.map { |user_id| { issue_id: issue_id, user_id: user_id } } - Gitlab::Database.main.bulk_insert(IssueAssignee.table_name, assignee_attrs) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert(IssueAssignee.table_name, assignee_attrs) # rubocop:disable Gitlab/BulkInsert end def build_label_attrs(issue_id, label_id) diff --git a/config/initializers/1_postgresql_only.rb b/config/initializers/1_postgresql_only.rb index 7bb851daa082b6500c54d8d8841635ec6f2a48d4..3be55255dddb2298f87024ee824d437c5e581dc2 100644 --- a/config/initializers/1_postgresql_only.rb +++ b/config/initializers/1_postgresql_only.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true raise "PostgreSQL is the only supported database from GitLab 12.1" unless - Gitlab::Database.main.postgresql? + ApplicationRecord.database.postgresql? Gitlab::Database.check_postgres_version_and_print_warning diff --git a/config/initializers/active_record_lifecycle.rb b/config/initializers/active_record_lifecycle.rb index 75991c9da351dd0af6fed4e2cc89cddfc9420acf..8d4b6d61abe34b16d5f262af3cd2949b2c27c48c 100644 --- a/config/initializers/active_record_lifecycle.rb +++ b/config/initializers/active_record_lifecycle.rb @@ -14,7 +14,7 @@ if defined?(ActiveRecord::Base) Gitlab::Cluster::LifecycleEvents.on_before_fork do - raise 'ActiveRecord connection not established. Unable to start.' unless Gitlab::Database.main.exists? + raise 'ActiveRecord connection not established. Unable to start.' unless ApplicationRecord.database.exists? # the following is highly recommended for Rails + "preload_app true" # as there's no need for the master process to hold a connection diff --git a/config/initializers/console_message.rb b/config/initializers/console_message.rb index 5e9e7a7a9af881ba9c2452f1c04d514621b918fa..3f98568c5002e39a9ed738486c6ecf0b3f55b5e3 100644 --- a/config/initializers/console_message.rb +++ b/config/initializers/console_message.rb @@ -10,8 +10,8 @@ puts " GitLab:".ljust(justify) + "#{Gitlab::VERSION} (#{Gitlab.revision}) #{Gitlab.ee? ? 'EE' : 'FOSS'}" puts " GitLab Shell:".ljust(justify) + "#{Gitlab::VersionInfo.parse(Gitlab::Shell.version)}" - if Gitlab::Database.main.exists? - puts " #{Gitlab::Database.main.human_adapter_name}:".ljust(justify) + Gitlab::Database.main.version + if ApplicationRecord.database.exists? + puts " #{ApplicationRecord.database.human_adapter_name}:".ljust(justify) + ApplicationRecord.database.version Gitlab.ee do if Gitlab::Geo.connected? && Gitlab::Geo.enabled? diff --git a/config/initializers/forbid_sidekiq_in_transactions.rb b/config/initializers/forbid_sidekiq_in_transactions.rb index ba5c1340b10e2de205c299418a68d48e1a64b079..e5e17672c4ed7bc0aeb39633920c55faadd7dd45 100644 --- a/config/initializers/forbid_sidekiq_in_transactions.rb +++ b/config/initializers/forbid_sidekiq_in_transactions.rb @@ -20,7 +20,7 @@ module ClassMethods module NoEnqueueingFromTransactions %i(perform_async perform_at perform_in).each do |name| define_method(name) do |*args| - if !Sidekiq::Worker.skip_transaction_check && Gitlab::Database.main.inside_transaction? + if !Sidekiq::Worker.skip_transaction_check && ApplicationRecord.inside_transaction? begin raise Sidekiq::Worker::EnqueueFromTransactionError, <<~MSG `#{self}.#{name}` cannot be called inside a transaction as this can lead to diff --git a/db/post_migrate/20200310215714_migrate_saml_identities_to_scim_identities.rb b/db/post_migrate/20200310215714_migrate_saml_identities_to_scim_identities.rb index 570eec53be372cc4913fa668b11f94d511a95f00..22b71a57f0464694fe626c525ec1ffca87ddb89c 100644 --- a/db/post_migrate/20200310215714_migrate_saml_identities_to_scim_identities.rb +++ b/db/post_migrate/20200310215714_migrate_saml_identities_to_scim_identities.rb @@ -20,7 +20,7 @@ def up record.attributes.extract!("extern_uid", "user_id", "group_id", "active", "created_at", "updated_at") end - Gitlab::Database.main.bulk_insert(:scim_identities, data_to_insert, on_conflict: :do_nothing) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert(:scim_identities, data_to_insert, on_conflict: :do_nothing) # rubocop:disable Gitlab/BulkInsert end end diff --git a/doc/development/verifying_database_capabilities.md b/doc/development/verifying_database_capabilities.md index c5e854701c2fdf6102526b9947a9866158717d01..bda9c68eae5f0eb639bc55d9b5859e412d33e279 100644 --- a/doc/development/verifying_database_capabilities.md +++ b/doc/development/verifying_database_capabilities.md @@ -12,13 +12,13 @@ necessary to add database (version) specific behavior. To facilitate this we have the following methods that you can use: -- `Gitlab::Database.main.version`: returns the PostgreSQL version number as a string +- `ApplicationRecord.database.version`: returns the PostgreSQL version number as a string in the format `X.Y.Z`. This allows you to write code such as: ```ruby -if Gitlab::Database.main.version.to_f >= 11.7 +if ApplicationRecord.database.version.to_f >= 11.7 run_really_fast_query else run_fast_query diff --git a/ee/app/models/ee/application_setting.rb b/ee/app/models/ee/application_setting.rb index de5ea96efa8c79ba8876d9982391d019deb61757..ba00ba3cd05186cd6b5740a3c6bf54c3615b5379 100644 --- a/ee/app/models/ee/application_setting.rb +++ b/ee/app/models/ee/application_setting.rb @@ -421,19 +421,19 @@ def mirror_capacity_threshold_less_than end def elasticsearch_indexing_column_exists? - ::Gitlab::Database.main.cached_column_exists?(:application_settings, :elasticsearch_indexing) + self.class.database.cached_column_exists?(:elasticsearch_indexing) end def elasticsearch_pause_indexing_column_exists? - ::Gitlab::Database.main.cached_column_exists?(:application_settings, :elasticsearch_pause_indexing) + self.class.database.cached_column_exists?(:elasticsearch_pause_indexing) end def elasticsearch_search_column_exists? - ::Gitlab::Database.main.cached_column_exists?(:application_settings, :elasticsearch_search) + self.class.database.cached_column_exists?(:elasticsearch_search) end def email_additional_text_column_exists? - ::Gitlab::Database.main.cached_column_exists?(:application_settings, :email_additional_text) + self.class.database.cached_column_exists?(:email_additional_text) end def check_geo_node_allowed_ips diff --git a/ee/app/models/elasticsearch_indexed_namespace.rb b/ee/app/models/elasticsearch_indexed_namespace.rb index e688157c6f54444c2234b43673501dac15b8588c..1d5b75cfff51f2ae7a00ec5cc5bd7e8d5a25a2f1 100644 --- a/ee/app/models/elasticsearch_indexed_namespace.rb +++ b/ee/app/models/elasticsearch_indexed_namespace.rb @@ -37,7 +37,7 @@ def self.index_first_n_namespaces_of_plan(plan, number_of_namespaces) { created_at: now, updated_at: now, namespace_id: id } end - Gitlab::Database.main.bulk_insert(table_name, insert_rows) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert(table_name, insert_rows) # rubocop:disable Gitlab/BulkInsert invalidate_elasticsearch_indexes_cache! jobs = batch_ids.map { |id| [id, :index] } diff --git a/ee/app/services/audit_events/bulk_insert_service.rb b/ee/app/services/audit_events/bulk_insert_service.rb index 107c4331827d7c82306eef16318c557ae26b26be..793f8cb7ea6367a98cda2f7b65659c9b65a2cc91 100644 --- a/ee/app/services/audit_events/bulk_insert_service.rb +++ b/ee/app/services/audit_events/bulk_insert_service.rb @@ -18,7 +18,7 @@ def execute return if collection.empty? collection.in_groups_of(BATCH_SIZE, false) do |services| - ::Gitlab::Database.main.bulk_insert(::AuditEvent.table_name, services.map(&:attributes)) # rubocop:disable Gitlab/BulkInsert + ::ApplicationRecord.legacy_bulk_insert(::AuditEvent.table_name, services.map(&:attributes)) # rubocop:disable Gitlab/BulkInsert services.each(&:log_security_event_to_file) end diff --git a/ee/app/services/iterations/cadences/create_iterations_in_advance_service.rb b/ee/app/services/iterations/cadences/create_iterations_in_advance_service.rb index e06722a27eb2be817cb28feae520772516f6b97a..3d63b839bae1130165bcf0c162ef4c46a4c1d035 100644 --- a/ee/app/services/iterations/cadences/create_iterations_in_advance_service.rb +++ b/ee/app/services/iterations/cadences/create_iterations_in_advance_service.rb @@ -15,7 +15,7 @@ def execute return ::ServiceResponse.error(message: _('Cadence is not automated'), http_status: 422) unless cadence.can_be_automated? update_existing_iterations! - ::Gitlab::Database.main.bulk_insert(Iteration.table_name, build_new_iterations) # rubocop:disable Gitlab/BulkInsert + ::ApplicationRecord.legacy_bulk_insert(Iteration.table_name, build_new_iterations) # rubocop:disable Gitlab/BulkInsert cadence.update!(last_run_date: compute_last_run_date) diff --git a/ee/app/services/iterations/roll_over_issues_service.rb b/ee/app/services/iterations/roll_over_issues_service.rb index 680295e33b8c96f4a92c3298e2dbbd1efd259e11..ac715c1572729b24ca1cccef4f32f69d16fc713a 100644 --- a/ee/app/services/iterations/roll_over_issues_service.rb +++ b/ee/app/services/iterations/roll_over_issues_service.rb @@ -20,8 +20,8 @@ def execute ApplicationRecord.transaction do issues.update_all(sprint_id: to_iteration.id, updated_at: rolled_over_at) - Gitlab::Database.main.bulk_insert(ResourceIterationEvent.table_name, remove_iteration_events) # rubocop:disable Gitlab/BulkInsert - Gitlab::Database.main.bulk_insert(ResourceIterationEvent.table_name, add_iteration_events) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert(ResourceIterationEvent.table_name, remove_iteration_events) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert(ResourceIterationEvent.table_name, add_iteration_events) # rubocop:disable Gitlab/BulkInsert end end diff --git a/ee/app/services/resource_events/change_weight_service.rb b/ee/app/services/resource_events/change_weight_service.rb index 50f151a2583b0bce47b19ed18da60dec86f8e0bc..83785f722c89647c8da091d0198a5133aa10c673 100644 --- a/ee/app/services/resource_events/change_weight_service.rb +++ b/ee/app/services/resource_events/change_weight_service.rb @@ -10,7 +10,7 @@ def initialize(resource, user) end def execute - ::Gitlab::Database.main.bulk_insert(ResourceWeightEvent.table_name, resource_weight_changes) # rubocop:disable Gitlab/BulkInsert + ::ApplicationRecord.legacy_bulk_insert(ResourceWeightEvent.table_name, resource_weight_changes) # rubocop:disable Gitlab/BulkInsert resource.expire_note_etag_cache Gitlab::UsageDataCounters::IssueActivityUniqueCounter.track_issue_weight_changed_action(author: user) diff --git a/ee/app/workers/geo/scheduler/scheduler_worker.rb b/ee/app/workers/geo/scheduler/scheduler_worker.rb index 87759eeddbc718286655fc6e9e0f8b3e9fb69a12..69f2d10e0d3e9c92255251696b95fcea84deb357 100644 --- a/ee/app/workers/geo/scheduler/scheduler_worker.rb +++ b/ee/app/workers/geo/scheduler/scheduler_worker.rb @@ -163,7 +163,7 @@ def update_jobs_in_progress def update_pending_resources if reload_queue? - @pending_resources = Gitlab::Database.main.geo_uncached_queries { load_pending_resources } + @pending_resources = Gitlab::Geo.uncached_queries { load_pending_resources } set_backoff_time! if should_apply_backoff? end end diff --git a/ee/lib/ee/gitlab/background_migration/generate_gitlab_subscriptions.rb b/ee/lib/ee/gitlab/background_migration/generate_gitlab_subscriptions.rb index a8fa0695914d267e9631075dd6f0e0137a44e0fe..56d0587e9ac28584006748fc6576559fad733ead 100644 --- a/ee/lib/ee/gitlab/background_migration/generate_gitlab_subscriptions.rb +++ b/ee/lib/ee/gitlab/background_migration/generate_gitlab_subscriptions.rb @@ -49,7 +49,7 @@ def perform(start_id, stop_id) } end - Gitlab::Database.main.bulk_insert(:gitlab_subscriptions, rows) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert(:gitlab_subscriptions, rows) # rubocop:disable Gitlab/BulkInsert end end end diff --git a/ee/lib/ee/gitlab/database/connection.rb b/ee/lib/ee/gitlab/database/connection.rb deleted file mode 100644 index d87f5bf521f35e55dab79d8e690602c8851381ae..0000000000000000000000000000000000000000 --- a/ee/lib/ee/gitlab/database/connection.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -module EE - module Gitlab - module Database - module Connection - extend ActiveSupport::Concern - - def geo_uncached_queries(&block) - raise 'No block given' unless block_given? - - scope.uncached do - if ::Gitlab::Geo.secondary? - Geo::TrackingBase.uncached(&block) - else - yield - end - end - end - end - end - end -end diff --git a/ee/lib/ee/gitlab/import_sources.rb b/ee/lib/ee/gitlab/import_sources.rb index 1b8d2a89e90b38635b1b444b76a5952253603623..e474d3761c66dadf02f56d888d3e813f83c634da 100644 --- a/ee/lib/ee/gitlab/import_sources.rb +++ b/ee/lib/ee/gitlab/import_sources.rb @@ -14,7 +14,7 @@ def ee_import_table # This method can be called/loaded before the database # has been created. With this guard clause we prevent querying # the License table until the table exists - return [] unless ::Gitlab::Database.main.cached_table_exists?('licenses') && + return [] unless License.database.cached_table_exists? && License.feature_available?(:custom_project_templates) [::Gitlab::ImportSources::ImportSource.new('gitlab_custom_project_template', diff --git a/ee/lib/ee/gitlab/middleware/read_only/controller.rb b/ee/lib/ee/gitlab/middleware/read_only/controller.rb index 386afd3fdb4906086502b0be98a89cb68d64ab90..1d799715fd227535590d259e06a6b025dc6c6cc7 100644 --- a/ee/lib/ee/gitlab/middleware/read_only/controller.rb +++ b/ee/lib/ee/gitlab/middleware/read_only/controller.rb @@ -83,7 +83,7 @@ def geo_node_update_route? action = route_hash[:action] if ALLOWLISTED_GEO_ROUTES[controller]&.include?(action) - ::Gitlab::Database.main.db_read_write? + ::ApplicationRecord.database.db_read_write? else ALLOWLISTED_GEO_ROUTES_TRACKING_DB[controller]&.include?(action) end diff --git a/ee/lib/gitlab/geo.rb b/ee/lib/gitlab/geo.rb index aedf1d12efc957f3f5d3c3c84f8e357d6497ffd3..e72abab8cb9c3d269884663a54a40ad8b5064b4f 100644 --- a/ee/lib/gitlab/geo.rb +++ b/ee/lib/gitlab/geo.rb @@ -213,5 +213,17 @@ def self.verification_max_capacity_per_replicator_class [1, capacity].max # at least 1 end + + def self.uncached_queries(&block) + raise 'No block given' unless block_given? + + ApplicationRecord.uncached do + if ::Gitlab::Geo.secondary? + ::Geo::TrackingBase.uncached(&block) + else + yield + end + end + end end end diff --git a/ee/lib/gitlab/geo/health_check.rb b/ee/lib/gitlab/geo/health_check.rb index f30b22a05bb1e3fb4072a974b762c886eeba3997..8fa4159243b4e981c0e8fb5829c7c58a126860e4 100644 --- a/ee/lib/gitlab/geo/health_check.rb +++ b/ee/lib/gitlab/geo/health_check.rb @@ -9,7 +9,7 @@ def perform_checks return '' unless Gitlab::Geo.secondary? return 'Geo database configuration file is missing.' unless Gitlab::Geo.geo_database_configured? return 'An existing tracking database cannot be reused.' if reusing_existing_tracking_database? - return 'Geo node has a database that is writable which is an indication it is not configured for replication with the primary node.' unless Gitlab::Database.main.db_read_only? + return 'Geo node has a database that is writable which is an indication it is not configured for replication with the primary node.' unless ApplicationRecord.database.db_read_only? return 'Geo node does not appear to be replicating the database from the primary node.' if replication_enabled? && !replication_working? return "Geo database version (#{database_version}) does not match latest migration (#{migration_version}).\nYou may have to run `gitlab-rake geo:db:migrate` as root on the secondary." unless database_migration_version_match? diff --git a/ee/spec/lib/ee/gitlab/database/connection_spec.rb b/ee/spec/lib/ee/gitlab/database/connection_spec.rb deleted file mode 100644 index aaf84757b4f2098bc60c395eca501a152f240098..0000000000000000000000000000000000000000 --- a/ee/spec/lib/ee/gitlab/database/connection_spec.rb +++ /dev/null @@ -1,61 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Database::Connection do - include ::EE::GeoHelpers - - let(:connection) { described_class.new } - - describe '#geo_uncached_queries' do - context 'when no block is given' do - it 'raises error' do - expect do - connection.geo_uncached_queries - end.to raise_error('No block given') - end - end - - context 'when the current node is a primary' do - let!(:primary) { create(:geo_node, :primary) } - - it 'wraps the block in an ActiveRecord::Base.uncached block' do - stub_current_geo_node(primary) - - expect(Geo::TrackingBase).not_to receive(:uncached) - expect(ActiveRecord::Base).to receive(:uncached).and_call_original - - expect do |b| - connection.geo_uncached_queries(&b) - end.to yield_control - end - end - - context 'when the current node is a secondary' do - let!(:primary) { create(:geo_node, :primary) } - let!(:secondary) { create(:geo_node) } - - it 'wraps the block in a Geo::TrackingBase.uncached block and an ActiveRecord::Base.uncached block' do - stub_current_geo_node(secondary) - - expect(Geo::TrackingBase).to receive(:uncached).and_call_original - expect(ActiveRecord::Base).to receive(:uncached).and_call_original - - expect do |b| - connection.geo_uncached_queries(&b) - end.to yield_control - end - end - - context 'when there is no current node' do - it 'wraps the block in an ActiveRecord::Base.uncached block' do - expect(Geo::TrackingBase).not_to receive(:uncached) - expect(ActiveRecord::Base).to receive(:uncached).and_call_original - - expect do |b| - connection.geo_uncached_queries(&b) - end.to yield_control - end - end - end -end diff --git a/ee/spec/lib/gitlab/geo/health_check_spec.rb b/ee/spec/lib/gitlab/geo/health_check_spec.rb index 440db576b2569f46dbb5a5a45b0ebb6cd5b65a60..970533faccd1da57723e6c44344fbc715c58af65 100644 --- a/ee/spec/lib/gitlab/geo/health_check_spec.rb +++ b/ee/spec/lib/gitlab/geo/health_check_spec.rb @@ -37,7 +37,7 @@ before do allow(Gitlab::Geo).to receive(:secondary?) { true } allow(Gitlab::Geo).to receive(:geo_database_configured?) { geo_database_configured } - allow(Gitlab::Database.main).to receive(:db_read_only?) { db_read_only } + allow(ApplicationRecord.database).to receive(:db_read_only?) { db_read_only } end context 'when the Geo tracking DB is not configured' do @@ -124,8 +124,8 @@ describe '#db_replication_lag_seconds' do before do query = 'SELECT CASE WHEN pg_last_wal_receive_lsn() = pg_last_wal_replay_lsn() THEN 0 ELSE EXTRACT (EPOCH FROM now() - pg_last_xact_replay_timestamp())::INTEGER END AS replication_lag' - allow(Gitlab::Database.main).to receive(:pg_last_wal_receive_lsn).and_return('pg_last_wal_receive_lsn') - allow(Gitlab::Database.main).to receive(:pg_last_wal_replay_lsn).and_return('pg_last_wal_replay_lsn') + allow(ApplicationRecord.database).to receive(:pg_last_wal_receive_lsn).and_return('pg_last_wal_receive_lsn') + allow(ApplicationRecord.database).to receive(:pg_last_wal_replay_lsn).and_return('pg_last_wal_replay_lsn') allow(ActiveRecord::Base).to receive_message_chain('connection.execute').with(query).and_return([{ 'replication_lag' => lag_in_seconds }]) end diff --git a/ee/spec/lib/gitlab/geo_spec.rb b/ee/spec/lib/gitlab/geo_spec.rb index b6923a3c671ae3cc54c035d42ed0873a87cd5466..73bf5a56b546e0c21e8f1c8cee37b17ed7a3e3e5 100644 --- a/ee/spec/lib/gitlab/geo_spec.rb +++ b/ee/spec/lib/gitlab/geo_spec.rb @@ -426,4 +426,51 @@ end end end + + describe '.uncached_queries' do + context 'when no block is given' do + it 'raises error' do + expect do + described_class.uncached_queries + end.to raise_error('No block given') + end + end + + context 'when the current node is a primary' do + it 'wraps the block in an ApplicationRecord.uncached block' do + stub_current_geo_node(primary_node) + + expect(Geo::TrackingBase).not_to receive(:uncached) + expect(ApplicationRecord).to receive(:uncached).and_call_original + + expect do |b| + described_class.uncached_queries(&b) + end.to yield_control + end + end + + context 'when the current node is a secondary' do + it 'wraps the block in a Geo::TrackingBase.uncached block and an ApplicationRecord.uncached block' do + stub_current_geo_node(secondary_node) + + expect(Geo::TrackingBase).to receive(:uncached).and_call_original + expect(ApplicationRecord).to receive(:uncached).and_call_original + + expect do |b| + described_class.uncached_queries(&b) + end.to yield_control + end + end + + context 'when there is no current node' do + it 'wraps the block in an ApplicationRecord.uncached block' do + expect(Geo::TrackingBase).not_to receive(:uncached) + expect(ApplicationRecord).to receive(:uncached).and_call_original + + expect do |b| + described_class.uncached_queries(&b) + end.to yield_control + end + end + end end diff --git a/lib/after_commit_queue.rb b/lib/after_commit_queue.rb index 2698d7adbd74593348d2680397285a88456c7281..cbeaea979512f45c7528c17e2fc2b3d8e4534ee1 100644 --- a/lib/after_commit_queue.rb +++ b/lib/after_commit_queue.rb @@ -15,7 +15,7 @@ def run_after_commit(&block) end def run_after_commit_or_now(&block) - if Gitlab::Database.main.inside_transaction? + if ApplicationRecord.inside_transaction? if ActiveRecord::Base.connection.current_transaction.records&.include?(self) run_after_commit(&block) else diff --git a/lib/feature.rb b/lib/feature.rb index d30594792c3aef07df6a54512700eb961a5950a0..8186fbc40fac2469f4de25c7ce1661a33c23c511 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -6,6 +6,8 @@ class Feature # Classes to override flipper table names class FlipperFeature < Flipper::Adapters::ActiveRecord::Feature + include DatabaseReflection + # Using `self.table_name` won't work. ActiveRecord bug? superclass.table_name = 'features' @@ -36,7 +38,7 @@ def get(key) end def persisted_names - return [] unless Gitlab::Database.main.exists? + return [] unless ApplicationRecord.database.exists? # This loads names of all stored feature flags # and returns a stable Set in the following order: @@ -73,7 +75,7 @@ def enabled?(key, thing = nil, type: :development, default_enabled: false) # During setup the database does not exist yet. So we haven't stored a value # for the feature yet and return the default. - return default_enabled unless Gitlab::Database.main.exists? + return default_enabled unless ApplicationRecord.database.exists? feature = get(key) diff --git a/lib/feature/gitaly.rb b/lib/feature/gitaly.rb index a061a83e79ccaaaa3e459e2fb5ebe34b4cf49628..a1f7dc0ee391a026d5c41b4d50d76a4d97ba9280 100644 --- a/lib/feature/gitaly.rb +++ b/lib/feature/gitaly.rb @@ -15,7 +15,7 @@ def enabled?(feature_flag, project = nil) def server_feature_flags(project = nil) # We need to check that both the DB connection and table exists - return {} unless ::Gitlab::Database.main.cached_table_exists?(FlipperFeature.table_name) + return {} unless FlipperFeature.database.cached_table_exists? Feature.persisted_names .select { |f| f.start_with?(PREFIX) } diff --git a/lib/gitlab/background_migration/backfill_project_repositories.rb b/lib/gitlab/background_migration/backfill_project_repositories.rb index a9eaeb0562dc5df9bde7cb2d434287a60b6a1336..05e2ed72fb3d4e9ea93621521d09dedea494af5c 100644 --- a/lib/gitlab/background_migration/backfill_project_repositories.rb +++ b/lib/gitlab/background_migration/backfill_project_repositories.rb @@ -189,7 +189,7 @@ def hashed_storage? end def perform(start_id, stop_id) - Gitlab::Database.main.bulk_insert(:project_repositories, project_repositories(start_id, stop_id)) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert(:project_repositories, project_repositories(start_id, stop_id)) # rubocop:disable Gitlab/BulkInsert end private diff --git a/lib/gitlab/background_migration/job_coordinator.rb b/lib/gitlab/background_migration/job_coordinator.rb index f03cf66fb66aa336c17bc95e0ef701cff4ec8358..7ebe523a83c988aef6c8f8970e4486b01eca9799 100644 --- a/lib/gitlab/background_migration/job_coordinator.rb +++ b/lib/gitlab/background_migration/job_coordinator.rb @@ -120,7 +120,10 @@ def initialize(database, worker_class) end def connection - @connection ||= Gitlab::Database.databases.fetch(database, Gitlab::Database.main).scope.connection + @connection ||= Gitlab::Database + .database_base_models + .fetch(database, Gitlab::Database::PRIMARY_DATABASE_NAME) + .connection end end end diff --git a/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb b/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb index 1c60473750d2390c33265874222c3efab9c35110..36a339c6b809be0bc2aa931a4caf90eeaf7f9406 100644 --- a/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb +++ b/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb @@ -34,7 +34,7 @@ def perform(start_id, stop_id) end end - Gitlab::Database.main.bulk_insert(TEMP_TABLE, fingerprints) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert(TEMP_TABLE, fingerprints) # rubocop:disable Gitlab/BulkInsert execute("ANALYZE #{TEMP_TABLE}") diff --git a/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb b/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb index 14c72bb4a72b5ad521a69531403a64032b8fe714..1e7924162c66bca9af29da46b2aa822d21f6de3d 100644 --- a/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb +++ b/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb @@ -65,7 +65,7 @@ def perform(start_id, stop_id) next if service_ids.empty? migrated_ids += service_ids - Gitlab::Database.main.bulk_insert(table, data) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert(table, data) # rubocop:disable Gitlab/BulkInsert end return if migrated_ids.empty? diff --git a/lib/gitlab/background_migration/populate_issue_email_participants.rb b/lib/gitlab/background_migration/populate_issue_email_participants.rb index 0a56ac1dae827edcb6571941bd8c9db0b21cc96a..2b959b81f45d0c84342e34ea5b213a5d881a997d 100644 --- a/lib/gitlab/background_migration/populate_issue_email_participants.rb +++ b/lib/gitlab/background_migration/populate_issue_email_participants.rb @@ -21,7 +21,7 @@ def perform(start_id, stop_id) } end - Gitlab::Database.main.bulk_insert(:issue_email_participants, rows, on_conflict: :do_nothing) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert(:issue_email_participants, rows, on_conflict: :do_nothing) # rubocop:disable Gitlab/BulkInsert end end end diff --git a/lib/gitlab/config_checker/external_database_checker.rb b/lib/gitlab/config_checker/external_database_checker.rb index a56f24136159eb676e62555260f2d132cb9f0c92..54320b7ff9ae66bdb925c9ae4e9f9c1d452ff59e 100644 --- a/lib/gitlab/config_checker/external_database_checker.rb +++ b/lib/gitlab/config_checker/external_database_checker.rb @@ -6,7 +6,7 @@ module ExternalDatabaseChecker extend self def check - return [] if Gitlab::Database.main.postgresql_minimum_supported_version? + return [] if ApplicationRecord.database.postgresql_minimum_supported_version? [ { @@ -15,7 +15,7 @@ def check '%{pg_version_minimum} is required for this version of GitLab. ' \ 'Please upgrade your environment to a supported PostgreSQL version, ' \ 'see %{pg_requirements_url} for details.') % { - pg_version_current: Gitlab::Database.main.version, + pg_version_current: ApplicationRecord.database.version, pg_version_minimum: Gitlab::Database::MINIMUM_POSTGRES_VERSION, pg_requirements_url: '<a href="https://docs.gitlab.com/ee/install/requirements.html#database">database requirements</a>' } diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index bfe3f06a56b1b86aa44222cec9005d22c2e45ff7..b9034cff447f5e7865d5a1f8a68866cd43f41141 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -85,7 +85,7 @@ def connect_to_db? active_db_connection = ActiveRecord::Base.connection.active? rescue false active_db_connection && - Gitlab::Database.main.cached_table_exists?('application_settings') + ApplicationSetting.database.cached_table_exists? rescue ActiveRecord::NoDatabaseError false end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 357ef48f3e06b92e1710aac36936c6d0781021e8..c83436f24b4e7518c110cd1306132cd3cf3a3205 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -60,18 +60,7 @@ def self.database_base_models # inherit from ApplicationRecord. main: ::ActiveRecord::Base, ci: ::Ci::ApplicationRecord.connection_class? ? ::Ci::ApplicationRecord : nil - }.compact.freeze - end - - def self.databases - @databases ||= database_base_models - .transform_values { |connection_class| Connection.new(connection_class) } - .with_indifferent_access - .freeze - end - - def self.main - databases[PRIMARY_DATABASE_NAME] + }.compact.with_indifferent_access.freeze end # We configure the database connection pool size automatically based on the @@ -110,8 +99,10 @@ def self.ci_database?(name) def self.check_postgres_version_and_print_warning return if Gitlab::Runtime.rails_runner? - databases.each do |name, connection| - next if connection.postgresql_minimum_supported_version? + database_base_models.each do |name, model| + database = Gitlab::Database::Reflection.new(model) + + next if database.postgresql_minimum_supported_version? Kernel.warn ERB.new(Rainbow.new.wrap(<<~EOS).red).result @@ -122,7 +113,7 @@ def self.check_postgres_version_and_print_warning â€â–ˆâ–ˆâ–ˆâ€â–ˆâ–ˆâ–ˆâ€â€â–ˆâ–ˆâ€ ██â€â–ˆâ–ˆâ€ ██â€â–ˆâ–ˆâ€ â€â–ˆâ–ˆâ–ˆâ–ˆâ€â–ˆâ–ˆâ€â–ˆâ–ˆâ€ â€â–ˆâ–ˆâ–ˆâ–ˆâ€â€â–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ–ˆâ€â€ ****************************************************************************** - You are using PostgreSQL #{connection.version} for the #{name} database, but PostgreSQL >= <%= Gitlab::Database::MINIMUM_POSTGRES_VERSION %> + You are using PostgreSQL #{database.version} for the #{name} database, but PostgreSQL >= <%= Gitlab::Database::MINIMUM_POSTGRES_VERSION %> is required for this version of GitLab. <% if Rails.env.development? || Rails.env.test? %> If using gitlab-development-kit, please find the relevant steps here: diff --git a/lib/gitlab/database/as_with_materialized.rb b/lib/gitlab/database/as_with_materialized.rb index 07809c5b59250717e34e72590f5816f60654037d..a04ea97117dd47be3437513d9fa1485c46723b8a 100644 --- a/lib/gitlab/database/as_with_materialized.rb +++ b/lib/gitlab/database/as_with_materialized.rb @@ -19,7 +19,7 @@ def initialize(left, right, materialized: true) # Note: to be deleted after the minimum PG version is set to 12.0 def self.materialized_supported? strong_memoize(:materialized_supported) do - Gitlab::Database.main.version.match?(/^1[2-9]\./) # version 12.x and above + ApplicationRecord.database.version.match?(/^1[2-9]\./) # version 12.x and above end end diff --git a/lib/gitlab/database/connection.rb b/lib/gitlab/database/connection.rb deleted file mode 100644 index 20b2126f371941d0417cd8b9b5db589de4b3cb7a..0000000000000000000000000000000000000000 --- a/lib/gitlab/database/connection.rb +++ /dev/null @@ -1,206 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - # Configuration settings and methods for interacting with a PostgreSQL - # database, with support for multiple databases. - class Connection - attr_reader :scope - - # Initializes a new `Database`. - # - # The `scope` argument must be an object (such as `ActiveRecord::Base`) - # that supports retrieving connections and connection pools. - def initialize(scope = ActiveRecord::Base) - @config = nil - @scope = scope - @version = nil - @open_transactions_baseline = 0 - end - - def config - # The result of this method must not be cached, as other methods may use - # it after making configuration changes and expect those changes to be - # present. For example, `disable_prepared_statements` expects the - # configuration settings to always be up to date. - # - # See the following for more information: - # - # - https://gitlab.com/gitlab-org/release/retrospectives/-/issues/39 - # - https://gitlab.com/gitlab-com/gl-infra/production/-/issues/5238 - scope.connection_db_config.configuration_hash.with_indifferent_access - end - - def pool_size - config[:pool] || Database.default_pool_size - end - - def username - config[:username] || ENV['USER'] - end - - def database_name - config[:database] - end - - def adapter_name - config[:adapter] - end - - def human_adapter_name - if postgresql? - 'PostgreSQL' - else - 'Unknown' - end - end - - def postgresql? - adapter_name.casecmp('postgresql') == 0 - end - - # Check whether the underlying database is in read-only mode - def db_read_only? - pg_is_in_recovery = - scope - .connection - .execute('SELECT pg_is_in_recovery()') - .first - .fetch('pg_is_in_recovery') - - Gitlab::Utils.to_boolean(pg_is_in_recovery) - end - - def db_read_write? - !db_read_only? - end - - def version - @version ||= database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1] - end - - def database_version - connection.execute("SELECT VERSION()").first['version'] - end - - def postgresql_minimum_supported_version? - version.to_f >= MINIMUM_POSTGRES_VERSION - end - - # Bulk inserts a number of rows into a table, optionally returning their - # IDs. - # - # table - The name of the table to insert the rows into. - # rows - An Array of Hash instances, each mapping the columns to their - # values. - # return_ids - When set to true the return value will be an Array of IDs of - # the inserted rows - # disable_quote - A key or an Array of keys to exclude from quoting (You - # become responsible for protection from SQL injection for - # these keys!) - # on_conflict - Defines an upsert. Values can be: :disabled (default) or - # :do_nothing - def bulk_insert(table, rows, return_ids: false, disable_quote: [], on_conflict: nil) - return if rows.empty? - - keys = rows.first.keys - columns = keys.map { |key| connection.quote_column_name(key) } - - disable_quote = Array(disable_quote).to_set - tuples = rows.map do |row| - keys.map do |k| - disable_quote.include?(k) ? row[k] : connection.quote(row[k]) - end - end - - sql = <<-EOF - INSERT INTO #{table} (#{columns.join(', ')}) - VALUES #{tuples.map { |tuple| "(#{tuple.join(', ')})" }.join(', ')} - EOF - - sql = "#{sql} ON CONFLICT DO NOTHING" if on_conflict == :do_nothing - - sql = "#{sql} RETURNING id" if return_ids - - result = connection.execute(sql) - - if return_ids - result.values.map { |tuple| tuple[0].to_i } - else - [] - end - end - - def cached_column_exists?(table_name, column_name) - connection - .schema_cache.columns_hash(table_name) - .has_key?(column_name.to_s) - end - - def cached_table_exists?(table_name) - exists? && connection.schema_cache.data_source_exists?(table_name) - end - - def exists? - # We can't _just_ check if `connection` raises an error, as it will - # point to a `ConnectionProxy`, and obtaining those doesn't involve any - # database queries. So instead we obtain the database version, which is - # cached after the first call. - connection.schema_cache.database_version - true - rescue StandardError - false - end - - def system_id - row = connection - .execute('SELECT system_identifier FROM pg_control_system()') - .first - - row['system_identifier'] - end - - def pg_wal_lsn_diff(location1, location2) - lsn1 = connection.quote(location1) - lsn2 = connection.quote(location2) - - query = <<-SQL.squish - SELECT pg_wal_lsn_diff(#{lsn1}, #{lsn2}) - AS result - SQL - - row = connection.select_all(query).first - row['result'] if row - end - - # inside_transaction? will return true if the caller is running within a - # transaction. Handles special cases when running inside a test - # environment, where tests may be wrapped in transactions - def inside_transaction? - base = Rails.env.test? ? @open_transactions_baseline : 0 - - scope.connection.open_transactions > base - end - - # These methods that access @open_transactions_baseline are not - # thread-safe. These are fine though because we only call these in - # RSpec's main thread. If we decide to run specs multi-threaded, we would - # need to use something like ThreadGroup to keep track of this value - def set_open_transactions_baseline - @open_transactions_baseline = scope.connection.open_transactions - end - - def reset_open_transactions_baseline - @open_transactions_baseline = 0 - end - - private - - def connection - scope.connection - end - end - end -end - -Gitlab::Database::Connection.prepend_mod_with('Gitlab::Database::Connection') diff --git a/lib/gitlab/database/each_database.rb b/lib/gitlab/database/each_database.rb index c9b07490594c3931a55e571deb6e81debea94baa..7c9e65e66919932aaf1c0713df526d9bf9622d5b 100644 --- a/lib/gitlab/database/each_database.rb +++ b/lib/gitlab/database/each_database.rb @@ -5,8 +5,8 @@ module Database module EachDatabase class << self def each_database_connection - Gitlab::Database.databases.each_pair do |connection_name, connection_wrapper| - connection = connection_wrapper.scope.connection + Gitlab::Database.database_base_models.each_pair do |connection_name, model| + connection = model.connection with_shared_connection(connection, connection_name) do yield connection, connection_name diff --git a/lib/gitlab/database/load_balancing/load_balancer.rb b/lib/gitlab/database/load_balancing/load_balancer.rb index 1dc39abcf1e1e3429784f719796f4fa3ad7c3a03..1e27bcfc55dcf8ef4b27c610aa19d4c391ae2f1c 100644 --- a/lib/gitlab/database/load_balancing/load_balancer.rb +++ b/lib/gitlab/database/load_balancing/load_balancer.rb @@ -263,6 +263,21 @@ def pool ) || raise(::ActiveRecord::ConnectionNotEstablished) end + def wal_diff(location1, location2) + read_write do |connection| + lsn1 = connection.quote(location1) + lsn2 = connection.quote(location2) + + query = <<-SQL.squish + SELECT pg_wal_lsn_diff(#{lsn1}, #{lsn2}) + AS result + SQL + + row = connection.select_all(query).first + row['result'] if row + end + end + private def ensure_caching! diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 642f8a0fde3be3082369dc6ebf688ad0e32b4369..7dce4fa0ce2e70c47801f22bd9ba82493ad1182e 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -1260,8 +1260,8 @@ def sidekiq_queue_length(queue_name) def check_trigger_permissions!(table) unless Grant.create_and_execute_trigger?(table) - dbname = Database.main.database_name - user = Database.main.username + dbname = ApplicationRecord.database.database_name + user = ApplicationRecord.database.username raise <<-EOF Your database user is not allowed to create, drop, or execute triggers on the @@ -1583,8 +1583,8 @@ def check_not_null_constraint_exists?(table, column, constraint_name: nil) def create_extension(extension) execute('CREATE EXTENSION IF NOT EXISTS %s' % extension) rescue ActiveRecord::StatementInvalid => e - dbname = Database.main.database_name - user = Database.main.username + dbname = ApplicationRecord.database.database_name + user = ApplicationRecord.database.username warn(<<~MSG) if e.to_s =~ /permission denied/ GitLab requires the PostgreSQL extension '#{extension}' installed in database '#{dbname}', but @@ -1611,8 +1611,8 @@ def create_extension(extension) def drop_extension(extension) execute('DROP EXTENSION IF EXISTS %s' % extension) rescue ActiveRecord::StatementInvalid => e - dbname = Database.main.database_name - user = Database.main.username + dbname = ApplicationRecord.database.database_name + user = ApplicationRecord.database.username warn(<<~MSG) if e.to_s =~ /permission denied/ This migration attempts to drop the PostgreSQL extension '#{extension}' diff --git a/lib/gitlab/database/reflection.rb b/lib/gitlab/database/reflection.rb new file mode 100644 index 0000000000000000000000000000000000000000..48a4de285411681b447e4e3124ce5d28b1c0b719 --- /dev/null +++ b/lib/gitlab/database/reflection.rb @@ -0,0 +1,115 @@ +# frozen_string_literal: true + +module Gitlab + module Database + # A class for reflecting upon a database and its settings, such as the + # adapter name, PostgreSQL version, and the presence of tables or columns. + class Reflection + attr_reader :model + + def initialize(model) + @model = model + @version = nil + end + + def config + # The result of this method must not be cached, as other methods may use + # it after making configuration changes and expect those changes to be + # present. For example, `disable_prepared_statements` expects the + # configuration settings to always be up to date. + # + # See the following for more information: + # + # - https://gitlab.com/gitlab-org/release/retrospectives/-/issues/39 + # - https://gitlab.com/gitlab-com/gl-infra/production/-/issues/5238 + model.connection_db_config.configuration_hash.with_indifferent_access + end + + def username + config[:username] || ENV['USER'] + end + + def database_name + config[:database] + end + + def adapter_name + config[:adapter] + end + + def human_adapter_name + if postgresql? + 'PostgreSQL' + else + 'Unknown' + end + end + + def postgresql? + adapter_name.casecmp('postgresql') == 0 + end + + # Check whether the underlying database is in read-only mode + def db_read_only? + pg_is_in_recovery = + connection + .execute('SELECT pg_is_in_recovery()') + .first + .fetch('pg_is_in_recovery') + + Gitlab::Utils.to_boolean(pg_is_in_recovery) + end + + def db_read_write? + !db_read_only? + end + + def version + @version ||= database_version.match(/\A(?:PostgreSQL |)([^\s]+).*\z/)[1] + end + + def database_version + connection.execute("SELECT VERSION()").first['version'] + end + + def postgresql_minimum_supported_version? + version.to_f >= MINIMUM_POSTGRES_VERSION + end + + def cached_column_exists?(column_name) + connection + .schema_cache.columns_hash(model.table_name) + .has_key?(column_name.to_s) + end + + def cached_table_exists? + exists? && connection.schema_cache.data_source_exists?(model.table_name) + end + + def exists? + # We can't _just_ check if `connection` raises an error, as it will + # point to a `ConnectionProxy`, and obtaining those doesn't involve any + # database queries. So instead we obtain the database version, which is + # cached after the first call. + connection.schema_cache.database_version + true + rescue StandardError + false + end + + def system_id + row = connection + .execute('SELECT system_identifier FROM pg_control_system()') + .first + + row['system_identifier'] + end + + private + + def connection + model.connection + end + end + end +end diff --git a/lib/gitlab/github_import/bulk_importing.rb b/lib/gitlab/github_import/bulk_importing.rb index 80f8f8bfbe267a2b9f1716e8b73e75b8eb2b1d17..28a39128ec9686ab35dd62f604f310491ad4bc9a 100644 --- a/lib/gitlab/github_import/bulk_importing.rb +++ b/lib/gitlab/github_import/bulk_importing.rb @@ -30,7 +30,7 @@ def build_database_rows(enum) # Bulk inserts the given rows into the database. def bulk_insert(model, rows, batch_size: 100) rows.each_slice(batch_size) do |slice| - Gitlab::Database.main.bulk_insert(model.table_name, slice) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert(model.table_name, slice) # rubocop:disable Gitlab/BulkInsert log_and_increment_counter(slice.size, :imported) end diff --git a/lib/gitlab/github_import/importer/diff_note_importer.rb b/lib/gitlab/github_import/importer/diff_note_importer.rb index fa60b2f1b7db3b1b23b27239ab87695d000c8ff1..0aa0896aa57d6c19f8ba9e91cc692cdab7a03c5a 100644 --- a/lib/gitlab/github_import/importer/diff_note_importer.rb +++ b/lib/gitlab/github_import/importer/diff_note_importer.rb @@ -70,7 +70,7 @@ def import_with_legacy_diff_note # To work around this we're using bulk_insert with a single row. This # allows us to efficiently insert data (even if it's just 1 row) # without having to use all sorts of hacks to disable callbacks. - Gitlab::Database.main.bulk_insert(LegacyDiffNote.table_name, [{ + ApplicationRecord.legacy_bulk_insert(LegacyDiffNote.table_name, [{ noteable_type: note.noteable_type, system: false, type: 'LegacyDiffNote', diff --git a/lib/gitlab/github_import/importer/issue_importer.rb b/lib/gitlab/github_import/importer/issue_importer.rb index f8665676ccfe2709b2a733d1844b87543e5f39f7..7f46615f17efebd9046f16c28a894dc1d1e6a5dd 100644 --- a/lib/gitlab/github_import/importer/issue_importer.rb +++ b/lib/gitlab/github_import/importer/issue_importer.rb @@ -75,7 +75,7 @@ def create_assignees(issue_id) end end - Gitlab::Database.main.bulk_insert(IssueAssignee.table_name, assignees) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert(IssueAssignee.table_name, assignees) # rubocop:disable Gitlab/BulkInsert end end end diff --git a/lib/gitlab/github_import/importer/label_links_importer.rb b/lib/gitlab/github_import/importer/label_links_importer.rb index b608bb48e38b927466b221651d324bf1af77361e..5e248c7cfc5f795e012ac0aa411871db394899df 100644 --- a/lib/gitlab/github_import/importer/label_links_importer.rb +++ b/lib/gitlab/github_import/importer/label_links_importer.rb @@ -40,7 +40,7 @@ def create_labels } end - Gitlab::Database.main.bulk_insert(LabelLink.table_name, rows) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert(LabelLink.table_name, rows) # rubocop:disable Gitlab/BulkInsert end def find_target_id diff --git a/lib/gitlab/github_import/importer/note_importer.rb b/lib/gitlab/github_import/importer/note_importer.rb index 1fd42a69fac580d338428811285d2df04cb7270f..2cc3a82dd9b1bde48e47c9a624adc406a02ef66b 100644 --- a/lib/gitlab/github_import/importer/note_importer.rb +++ b/lib/gitlab/github_import/importer/note_importer.rb @@ -37,7 +37,7 @@ def execute # We're using bulk_insert here so we can bypass any validations and # callbacks. Running these would result in a lot of unnecessary SQL # queries being executed when importing large projects. - Gitlab::Database.main.bulk_insert(Note.table_name, [attributes]) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert(Note.table_name, [attributes]) # rubocop:disable Gitlab/BulkInsert rescue ActiveRecord::InvalidForeignKey # It's possible the project and the issue have been deleted since # scheduling this job. In this case we'll just skip creating the note. diff --git a/lib/gitlab/import/database_helpers.rb b/lib/gitlab/import/database_helpers.rb index e73c3afe9bd7efbf4c6705541d79dcbdfc41cd7c..96490db0c0777e16a699737268c1ad611098c841 100644 --- a/lib/gitlab/import/database_helpers.rb +++ b/lib/gitlab/import/database_helpers.rb @@ -11,8 +11,8 @@ def insert_and_return_id(attributes, relation) # We use bulk_insert here so we can bypass any queries executed by # callbacks or validation rules, as doing this wouldn't scale when # importing very large projects. - result = Gitlab::Database.main # rubocop:disable Gitlab/BulkInsert - .bulk_insert(relation.table_name, [attributes], return_ids: true) + result = ApplicationRecord # rubocop:disable Gitlab/BulkInsert + .legacy_bulk_insert(relation.table_name, [attributes], return_ids: true) result.first end diff --git a/lib/gitlab/language_detection.rb b/lib/gitlab/language_detection.rb index fc9fb5caa09bd3ce48e33b9a5609d481cdbc196f..6f7fa9fe03b12721319f2e504c8abcd8c8b873e3 100644 --- a/lib/gitlab/language_detection.rb +++ b/lib/gitlab/language_detection.rb @@ -18,7 +18,7 @@ def language_color(name) end # Newly detected languages, returned in a structure accepted by - # Gitlab::Database.main.bulk_insert + # ApplicationRecord.legacy_bulk_insert def insertions(programming_languages) lang_to_id = programming_languages.to_h { |p| [p.name, p.id] } diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb index 818fe96aa9d2690d05992fcef6409af10c9ccab3..b389ac368dbca91197d2ee3448c7305f374c433a 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb @@ -208,7 +208,12 @@ def job_wal_locations end def pg_wal_lsn_diff(connection_name) - Gitlab::Database.databases[connection_name].pg_wal_lsn_diff(job_wal_locations[connection_name], existing_wal_locations[connection_name]) + model = Gitlab::Database.database_base_models[connection_name] + + model.connection.load_balancer.wal_diff( + job_wal_locations[connection_name], + existing_wal_locations[connection_name] + ) end def strategy diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 5fa843f3f1fe6f78576e4646ee14bfb8a3979cdd..9c9ecef01bc4bdb3f93ab867edff0f257173eacb 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -296,9 +296,11 @@ def components_usage_data version: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.container_registry_version } }, database: { - adapter: alt_usage_data { Gitlab::Database.main.adapter_name }, - version: alt_usage_data { Gitlab::Database.main.version }, - pg_system_id: alt_usage_data { Gitlab::Database.main.system_id } + # rubocop: disable UsageData/LargeTable + adapter: alt_usage_data { ApplicationRecord.database.adapter_name }, + version: alt_usage_data { ApplicationRecord.database.version }, + pg_system_id: alt_usage_data { ApplicationRecord.database.system_id } + # rubocop: enable UsageData/LargeTable }, mail: { smtp_server: alt_usage_data { ActionMailer::Base.smtp_settings[:address] } diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 730c140ee4ec0e8a9f3077f28d7fd6164a77d11b..e83c4cbdb399f9c6122c46f4b857bbb5149b06b9 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -182,9 +182,9 @@ namespace :gitlab do desc 'Enqueue an index for reindexing' task :enqueue_reindexing_action, [:index_name, :database] => :environment do |_, args| - connection = Gitlab::Database.databases[args.fetch(:database, Gitlab::Database::PRIMARY_DATABASE_NAME)] + model = Gitlab::Database.database_base_models[args.fetch(:database, Gitlab::Database::PRIMARY_DATABASE_NAME)] - Gitlab::Database::SharedModel.using_connection(connection.scope.connection) do + Gitlab::Database::SharedModel.using_connection(model.connection) do queued_action = Gitlab::Database::PostgresIndex.find(args[:index_name]).queued_reindexing_actions.create! puts "Queued reindexing action: #{queued_action}" diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index 68395d10d24ba3573b9904a26497818974a24168..02764b5d46f529ac18e08d14cc7a1ec68d1241f8 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -68,8 +68,8 @@ namespace :gitlab do puts "Version:\t#{Gitlab::VERSION}" puts "Revision:\t#{Gitlab.revision}" puts "Directory:\t#{Rails.root}" - puts "DB Adapter:\t#{Gitlab::Database.main.human_adapter_name}" - puts "DB Version:\t#{Gitlab::Database.main.version}" + puts "DB Adapter:\t#{ApplicationRecord.database.human_adapter_name}" + puts "DB Version:\t#{ApplicationRecord.database.version}" puts "URL:\t\t#{Gitlab.config.gitlab.url}" puts "HTTP Clone URL:\t#{http_clone_url}" puts "SSH Clone URL:\t#{ssh_clone_url}" diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake index fb9f9b9fe677998bd96885cd852df0e673694eb7..eb5eeed531ff2e574ec2559940c8e7c9df0d892b 100644 --- a/lib/tasks/gitlab/storage.rake +++ b/lib/tasks/gitlab/storage.rake @@ -170,7 +170,7 @@ namespace :gitlab do inverval = (ENV['MAX_DATABASE_CONNECTION_CHECK_INTERVAL'] || 10).to_f attempts.to_i.times do - unless Gitlab::Database.main.exists? + unless ApplicationRecord.database.exists? puts "Waiting until database is ready before continuing...".color(:yellow) sleep inverval end diff --git a/rubocop/cop/gitlab/bulk_insert.rb b/rubocop/cop/gitlab/bulk_insert.rb index 4c8c232043f77088d754328c3d8c1a0bfaf51c5d..baaefc2533ce4410a53f0078c8be74cd36373b10 100644 --- a/rubocop/cop/gitlab/bulk_insert.rb +++ b/rubocop/cop/gitlab/bulk_insert.rb @@ -3,13 +3,13 @@ module RuboCop module Cop module Gitlab - # Cop that disallows the use of `Gitlab::Database.main.bulk_insert`, in favour of using + # Cop that disallows the use of `legacy_bulk_insert`, in favour of using # the `BulkInsertSafe` module. class BulkInsert < RuboCop::Cop::Cop - MSG = 'Use the `BulkInsertSafe` concern, instead of using `Gitlab::Database.main.bulk_insert`. See https://docs.gitlab.com/ee/development/insert_into_tables_in_batches.html' + MSG = 'Use the `BulkInsertSafe` concern, instead of using `LegacyBulkInsert.bulk_insert`. See https://docs.gitlab.com/ee/development/insert_into_tables_in_batches.html' def_node_matcher :raw_union?, <<~PATTERN - (send (send (const (const _ :Gitlab) :Database) :main) :bulk_insert ...) + (send _ :legacy_bulk_insert ...) PATTERN def on_send(node) diff --git a/spec/factories/design_management/designs.rb b/spec/factories/design_management/designs.rb index c23a67fe95bf4d3f24e46a2114c9204dfc210d6b..56a1b55b9695367dc43a0be77621c4bc6bc60d33 100644 --- a/spec/factories/design_management/designs.rb +++ b/spec/factories/design_management/designs.rb @@ -39,7 +39,7 @@ sha = commit_version[action] version = DesignManagement::Version.new(sha: sha, issue: issue, author: evaluator.author) version.save!(validate: false) # We need it to have an ID, validate later - Gitlab::Database.main.bulk_insert(dv_table_name, [action.row_attrs(version)]) # rubocop:disable Gitlab/BulkInsert + ApplicationRecord.legacy_bulk_insert(dv_table_name, [action.row_attrs(version)]) # rubocop:disable Gitlab/BulkInsert end # always a creation diff --git a/spec/initializers/database_config_spec.rb b/spec/initializers/database_config_spec.rb index 147efd5523a2dfffb8b16a91ce15165b8dcaa35c..230f12967603e66074b634ef4023770e8151706e 100644 --- a/spec/initializers/database_config_spec.rb +++ b/spec/initializers/database_config_spec.rb @@ -8,11 +8,11 @@ end it 'retains the correct database name for the connection' do - previous_db_name = Gitlab::Database.main.scope.connection.pool.db_config.name + previous_db_name = ApplicationRecord.connection.pool.db_config.name subject - expect(Gitlab::Database.main.scope.connection.pool.db_config.name).to eq(previous_db_name) + expect(ApplicationRecord.connection.pool.db_config.name).to eq(previous_db_name) end it 'does not overwrite custom pool settings' do diff --git a/spec/lib/feature/gitaly_spec.rb b/spec/lib/feature/gitaly_spec.rb index 311589c325366db7c8853db201426676dd345d1e..ed80e31e3cdfca54531166c27131a0a494a13638 100644 --- a/spec/lib/feature/gitaly_spec.rb +++ b/spec/lib/feature/gitaly_spec.rb @@ -78,7 +78,9 @@ context 'when table does not exist' do before do - allow(::Gitlab::Database.main).to receive(:cached_table_exists?).and_return(false) + allow(Feature::FlipperFeature.database) + .to receive(:cached_table_exists?) + .and_return(false) end it 'returns an empty Hash' do diff --git a/spec/lib/feature_spec.rb b/spec/lib/feature_spec.rb index dce2eedf6c3708990bbeb7cee0df669a47d398bc..58e7292c12568e12557109fa17731fc1ce5ee3cc 100644 --- a/spec/lib/feature_spec.rb +++ b/spec/lib/feature_spec.rb @@ -314,7 +314,7 @@ context 'when database exists' do before do - allow(Gitlab::Database.main).to receive(:exists?).and_return(true) + allow(ApplicationRecord.database).to receive(:exists?).and_return(true) end it 'checks the persisted status and returns false' do @@ -326,7 +326,7 @@ context 'when database does not exist' do before do - allow(Gitlab::Database.main).to receive(:exists?).and_return(false) + allow(ApplicationRecord.database).to receive(:exists?).and_return(false) end it 'returns false without checking the status in the database' do diff --git a/spec/lib/gitlab/background_migration/job_coordinator_spec.rb b/spec/lib/gitlab/background_migration/job_coordinator_spec.rb index 8b5fb03820f86178bacb69c1bf82d4eaffdbfbc7..5e029f304c90b8e4f903be0c2af8520e988c4431 100644 --- a/spec/lib/gitlab/background_migration/job_coordinator_spec.rb +++ b/spec/lib/gitlab/background_migration/job_coordinator_spec.rb @@ -42,7 +42,7 @@ describe '#with_shared_connection' do it 'yields to the block after properly configuring SharedModel' do expect(Gitlab::Database::SharedModel).to receive(:using_connection) - .with(Gitlab::Database.main.scope.connection).and_yield + .with(ActiveRecord::Base.connection).and_yield expect { |b| coordinator.with_shared_connection(&b) }.to yield_with_no_args end diff --git a/spec/lib/gitlab/config_checker/external_database_checker_spec.rb b/spec/lib/gitlab/config_checker/external_database_checker_spec.rb index 5a4e9001ac92bd7f509a2418fe1d90fa2eb51d73..933b6d6be9e6902af115f00a13df9011822c1e36 100644 --- a/spec/lib/gitlab/config_checker/external_database_checker_spec.rb +++ b/spec/lib/gitlab/config_checker/external_database_checker_spec.rb @@ -8,7 +8,7 @@ context 'when database meets minimum supported version' do before do - allow(Gitlab::Database.main).to receive(:postgresql_minimum_supported_version?).and_return(true) + allow(ApplicationRecord.database).to receive(:postgresql_minimum_supported_version?).and_return(true) end it { is_expected.to be_empty } @@ -16,7 +16,7 @@ context 'when database does not meet minimum supported version' do before do - allow(Gitlab::Database.main).to receive(:postgresql_minimum_supported_version?).and_return(false) + allow(ApplicationRecord.database).to receive(:postgresql_minimum_supported_version?).and_return(false) end let(:notice_deprecated_database) do @@ -26,7 +26,7 @@ '%{pg_version_minimum} is required for this version of GitLab. ' \ 'Please upgrade your environment to a supported PostgreSQL version, ' \ 'see %{pg_requirements_url} for details.') % { - pg_version_current: Gitlab::Database.main.version, + pg_version_current: ApplicationRecord.database.version, pg_version_minimum: Gitlab::Database::MINIMUM_POSTGRES_VERSION, pg_requirements_url: '<a href="https://docs.gitlab.com/ee/install/requirements.html#database">database requirements</a>' } diff --git a/spec/lib/gitlab/database/connection_spec.rb b/spec/lib/gitlab/database/connection_spec.rb deleted file mode 100644 index 32a262964db82c9028d22707dfaa5e5945c52562..0000000000000000000000000000000000000000 --- a/spec/lib/gitlab/database/connection_spec.rb +++ /dev/null @@ -1,384 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::Database::Connection do - let(:connection) { described_class.new } - - describe '#config' do - it 'returns a HashWithIndifferentAccess' do - expect(connection.config).to be_an_instance_of(HashWithIndifferentAccess) - end - - it 'returns a default pool size' do - expect(connection.config) - .to include(pool: Gitlab::Database.default_pool_size) - end - - it 'does not cache its results' do - a = connection.config - b = connection.config - - expect(a).not_to equal(b) - end - end - - describe '#pool_size' do - context 'when no explicit size is configured' do - it 'returns the default pool size' do - expect(connection).to receive(:config).and_return({ pool: nil }) - - expect(connection.pool_size).to eq(Gitlab::Database.default_pool_size) - end - end - - context 'when an explicit pool size is set' do - it 'returns the pool size' do - expect(connection).to receive(:config).and_return({ pool: 4 }) - - expect(connection.pool_size).to eq(4) - end - end - end - - describe '#username' do - context 'when a username is set' do - it 'returns the username' do - allow(connection).to receive(:config).and_return(username: 'bob') - - expect(connection.username).to eq('bob') - end - end - - context 'when a username is not set' do - it 'returns the value of the USER environment variable' do - allow(connection).to receive(:config).and_return(username: nil) - allow(ENV).to receive(:[]).with('USER').and_return('bob') - - expect(connection.username).to eq('bob') - end - end - end - - describe '#database_name' do - it 'returns the name of the database' do - allow(connection).to receive(:config).and_return(database: 'test') - - expect(connection.database_name).to eq('test') - end - end - - describe '#adapter_name' do - it 'returns the database adapter name' do - allow(connection).to receive(:config).and_return(adapter: 'test') - - expect(connection.adapter_name).to eq('test') - end - end - - describe '#human_adapter_name' do - context 'when the adapter is PostgreSQL' do - it 'returns PostgreSQL' do - allow(connection).to receive(:config).and_return(adapter: 'postgresql') - - expect(connection.human_adapter_name).to eq('PostgreSQL') - end - end - - context 'when the adapter is not PostgreSQL' do - it 'returns Unknown' do - allow(connection).to receive(:config).and_return(adapter: 'kittens') - - expect(connection.human_adapter_name).to eq('Unknown') - end - end - end - - describe '#postgresql?' do - context 'when using PostgreSQL' do - it 'returns true' do - allow(connection).to receive(:adapter_name).and_return('PostgreSQL') - - expect(connection.postgresql?).to eq(true) - end - end - - context 'when not using PostgreSQL' do - it 'returns false' do - allow(connection).to receive(:adapter_name).and_return('MySQL') - - expect(connection.postgresql?).to eq(false) - end - end - end - - describe '#db_read_only?' do - it 'detects a read-only database' do - allow(connection.scope.connection) - .to receive(:execute) - .with('SELECT pg_is_in_recovery()') - .and_return([{ "pg_is_in_recovery" => "t" }]) - - expect(connection.db_read_only?).to be_truthy - end - - it 'detects a read-only database' do - allow(connection.scope.connection) - .to receive(:execute) - .with('SELECT pg_is_in_recovery()') - .and_return([{ "pg_is_in_recovery" => true }]) - - expect(connection.db_read_only?).to be_truthy - end - - it 'detects a read-write database' do - allow(connection.scope.connection) - .to receive(:execute) - .with('SELECT pg_is_in_recovery()') - .and_return([{ "pg_is_in_recovery" => "f" }]) - - expect(connection.db_read_only?).to be_falsey - end - - it 'detects a read-write database' do - allow(connection.scope.connection) - .to receive(:execute) - .with('SELECT pg_is_in_recovery()') - .and_return([{ "pg_is_in_recovery" => false }]) - - expect(connection.db_read_only?).to be_falsey - end - end - - describe '#db_read_write?' do - it 'detects a read-only database' do - allow(connection.scope.connection) - .to receive(:execute) - .with('SELECT pg_is_in_recovery()') - .and_return([{ "pg_is_in_recovery" => "t" }]) - - expect(connection.db_read_write?).to eq(false) - end - - it 'detects a read-only database' do - allow(connection.scope.connection) - .to receive(:execute) - .with('SELECT pg_is_in_recovery()') - .and_return([{ "pg_is_in_recovery" => true }]) - - expect(connection.db_read_write?).to eq(false) - end - - it 'detects a read-write database' do - allow(connection.scope.connection) - .to receive(:execute) - .with('SELECT pg_is_in_recovery()') - .and_return([{ "pg_is_in_recovery" => "f" }]) - - expect(connection.db_read_write?).to eq(true) - end - - it 'detects a read-write database' do - allow(connection.scope.connection) - .to receive(:execute) - .with('SELECT pg_is_in_recovery()') - .and_return([{ "pg_is_in_recovery" => false }]) - - expect(connection.db_read_write?).to eq(true) - end - end - - describe '#version' do - around do |example| - connection.instance_variable_set(:@version, nil) - example.run - connection.instance_variable_set(:@version, nil) - end - - context "on postgresql" do - it "extracts the version number" do - allow(connection) - .to receive(:database_version) - .and_return("PostgreSQL 9.4.4 on x86_64-apple-darwin14.3.0") - - expect(connection.version).to eq '9.4.4' - end - end - - it 'memoizes the result' do - count = ActiveRecord::QueryRecorder - .new { 2.times { connection.version } } - .count - - expect(count).to eq(1) - end - end - - describe '#postgresql_minimum_supported_version?' do - it 'returns false when using PostgreSQL 10' do - allow(connection).to receive(:version).and_return('10') - - expect(connection.postgresql_minimum_supported_version?).to eq(false) - end - - it 'returns false when using PostgreSQL 11' do - allow(connection).to receive(:version).and_return('11') - - expect(connection.postgresql_minimum_supported_version?).to eq(false) - end - - it 'returns true when using PostgreSQL 12' do - allow(connection).to receive(:version).and_return('12') - - expect(connection.postgresql_minimum_supported_version?).to eq(true) - end - end - - describe '#bulk_insert' do - before do - allow(connection).to receive(:connection).and_return(dummy_connection) - allow(dummy_connection).to receive(:quote_column_name, &:itself) - allow(dummy_connection).to receive(:quote, &:itself) - allow(dummy_connection).to receive(:execute) - end - - let(:dummy_connection) { double(:connection) } - - let(:rows) do - [ - { a: 1, b: 2, c: 3 }, - { c: 6, a: 4, b: 5 } - ] - end - - it 'does nothing with empty rows' do - expect(dummy_connection).not_to receive(:execute) - - connection.bulk_insert('test', []) - end - - it 'uses the ordering from the first row' do - expect(dummy_connection).to receive(:execute) do |sql| - expect(sql).to include('(1, 2, 3)') - expect(sql).to include('(4, 5, 6)') - end - - connection.bulk_insert('test', rows) - end - - it 'quotes column names' do - expect(dummy_connection).to receive(:quote_column_name).with(:a) - expect(dummy_connection).to receive(:quote_column_name).with(:b) - expect(dummy_connection).to receive(:quote_column_name).with(:c) - - connection.bulk_insert('test', rows) - end - - it 'quotes values' do - 1.upto(6) do |i| - expect(dummy_connection).to receive(:quote).with(i) - end - - connection.bulk_insert('test', rows) - end - - it 'does not quote values of a column in the disable_quote option' do - [1, 2, 4, 5].each do |i| - expect(dummy_connection).to receive(:quote).with(i) - end - - connection.bulk_insert('test', rows, disable_quote: :c) - end - - it 'does not quote values of columns in the disable_quote option' do - [2, 5].each do |i| - expect(dummy_connection).to receive(:quote).with(i) - end - - connection.bulk_insert('test', rows, disable_quote: [:a, :c]) - end - - it 'handles non-UTF-8 data' do - expect { connection.bulk_insert('test', [{ a: "\255" }]) }.not_to raise_error - end - - context 'when using PostgreSQL' do - it 'allows the returning of the IDs of the inserted rows' do - result = double(:result, values: [['10']]) - - expect(dummy_connection) - .to receive(:execute) - .with(/RETURNING id/) - .and_return(result) - - ids = connection - .bulk_insert('test', [{ number: 10 }], return_ids: true) - - expect(ids).to eq([10]) - end - - it 'allows setting the upsert to do nothing' do - expect(dummy_connection) - .to receive(:execute) - .with(/ON CONFLICT DO NOTHING/) - - connection - .bulk_insert('test', [{ number: 10 }], on_conflict: :do_nothing) - end - end - end - - describe '#cached_column_exists?' do - it 'only retrieves the data from the schema cache' do - queries = ActiveRecord::QueryRecorder.new do - 2.times do - expect(connection.cached_column_exists?(:projects, :id)).to be_truthy - expect(connection.cached_column_exists?(:projects, :bogus_column)).to be_falsey - end - end - - expect(queries.count).to eq(0) - end - end - - describe '#cached_table_exists?' do - it 'only retrieves the data from the schema cache' do - queries = ActiveRecord::QueryRecorder.new do - 2.times do - expect(connection.cached_table_exists?(:projects)).to be_truthy - expect(connection.cached_table_exists?(:bogus_table_name)).to be_falsey - end - end - - expect(queries.count).to eq(0) - end - - it 'returns false when database does not exist' do - expect(connection.scope).to receive(:connection) do - raise ActiveRecord::NoDatabaseError, 'broken' - end - - expect(connection.cached_table_exists?(:projects)).to be(false) - end - end - - describe '#exists?' do - it 'returns true if the database exists' do - expect(connection.exists?).to be(true) - end - - it "returns false if the database doesn't exist" do - expect(connection.scope.connection.schema_cache) - .to receive(:database_version) - .and_raise(ActiveRecord::NoDatabaseError) - - expect(connection.exists?).to be(false) - end - end - - describe '#system_id' do - it 'returns the PostgreSQL system identifier' do - expect(connection.system_id).to be_an_instance_of(Integer) - end - end -end diff --git a/spec/lib/gitlab/database/each_database_spec.rb b/spec/lib/gitlab/database/each_database_spec.rb index 8523249d804ad5f7347ff308458707922489a5a2..9327fc4ff78d11f72e27b19a1096402200b18f6f 100644 --- a/spec/lib/gitlab/database/each_database_spec.rb +++ b/spec/lib/gitlab/database/each_database_spec.rb @@ -3,9 +3,9 @@ require 'spec_helper' RSpec.describe Gitlab::Database::EachDatabase do - describe '.each_database' do + describe '.each_database_connection' do let(:expected_connections) do - Gitlab::Database.databases.map { |name, wrapper| [wrapper.scope.connection, name] } + Gitlab::Database.database_base_models.map { |name, model| [model.connection, name] } end it 'yields each connection after connecting SharedModel' do diff --git a/spec/lib/gitlab/database/load_balancing/load_balancer_spec.rb b/spec/lib/gitlab/database/load_balancing/load_balancer_spec.rb index 957a2da7f9c0ece4e60fd5ffd4988696d624ac90..37b8372912533aabd441c0e072027151041bec1b 100644 --- a/spec/lib/gitlab/database/load_balancing/load_balancer_spec.rb +++ b/spec/lib/gitlab/database/load_balancing/load_balancer_spec.rb @@ -542,4 +542,17 @@ def with_replica_pool(*args) end end end + + describe '#wal_diff' do + it 'returns the diff between two write locations' do + loc1 = lb.send(:get_write_location, lb.pool.connection) + + create(:user) # This ensures we get a new WAL location + + loc2 = lb.send(:get_write_location, lb.pool.connection) + diff = lb.wal_diff(loc2, loc1) + + expect(diff).to be_positive + end + end end diff --git a/spec/lib/gitlab/database/partitioning/partition_manager_spec.rb b/spec/lib/gitlab/database/partitioning/partition_manager_spec.rb index 7c4cfcfb3a99fccef84986cfb7295c62f4d2ee1a..1c6f5c5c694bb36d0da24f9f96103938ba340c96 100644 --- a/spec/lib/gitlab/database/partitioning/partition_manager_spec.rb +++ b/spec/lib/gitlab/database/partitioning/partition_manager_spec.rb @@ -195,7 +195,7 @@ def num_tables end # Postgres 11 does not support foreign keys to partitioned tables - if Gitlab::Database.main.version.to_f >= 12 + if ApplicationRecord.database.version.to_f >= 12 context 'when the model is the target of a foreign key' do before do connection.execute(<<~SQL) diff --git a/spec/lib/gitlab/database/reflection_spec.rb b/spec/lib/gitlab/database/reflection_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..7c3d797817d938292557a2b879f4a7ad6b5043ca --- /dev/null +++ b/spec/lib/gitlab/database/reflection_spec.rb @@ -0,0 +1,280 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Database::Reflection do + let(:database) { described_class.new(ApplicationRecord) } + + describe '#username' do + context 'when a username is set' do + it 'returns the username' do + allow(database).to receive(:config).and_return(username: 'bob') + + expect(database.username).to eq('bob') + end + end + + context 'when a username is not set' do + it 'returns the value of the USER environment variable' do + allow(database).to receive(:config).and_return(username: nil) + allow(ENV).to receive(:[]).with('USER').and_return('bob') + + expect(database.username).to eq('bob') + end + end + end + + describe '#database_name' do + it 'returns the name of the database' do + allow(database).to receive(:config).and_return(database: 'test') + + expect(database.database_name).to eq('test') + end + end + + describe '#adapter_name' do + it 'returns the database adapter name' do + allow(database).to receive(:config).and_return(adapter: 'test') + + expect(database.adapter_name).to eq('test') + end + end + + describe '#human_adapter_name' do + context 'when the adapter is PostgreSQL' do + it 'returns PostgreSQL' do + allow(database).to receive(:config).and_return(adapter: 'postgresql') + + expect(database.human_adapter_name).to eq('PostgreSQL') + end + end + + context 'when the adapter is not PostgreSQL' do + it 'returns Unknown' do + allow(database).to receive(:config).and_return(adapter: 'kittens') + + expect(database.human_adapter_name).to eq('Unknown') + end + end + end + + describe '#postgresql?' do + context 'when using PostgreSQL' do + it 'returns true' do + allow(database).to receive(:adapter_name).and_return('PostgreSQL') + + expect(database.postgresql?).to eq(true) + end + end + + context 'when not using PostgreSQL' do + it 'returns false' do + allow(database).to receive(:adapter_name).and_return('MySQL') + + expect(database.postgresql?).to eq(false) + end + end + end + + describe '#db_read_only?' do + it 'detects a read-only database' do + allow(database.model.connection) + .to receive(:execute) + .with('SELECT pg_is_in_recovery()') + .and_return([{ "pg_is_in_recovery" => "t" }]) + + expect(database.db_read_only?).to be_truthy + end + + it 'detects a read-only database' do + allow(database.model.connection) + .to receive(:execute) + .with('SELECT pg_is_in_recovery()') + .and_return([{ "pg_is_in_recovery" => true }]) + + expect(database.db_read_only?).to be_truthy + end + + it 'detects a read-write database' do + allow(database.model.connection) + .to receive(:execute) + .with('SELECT pg_is_in_recovery()') + .and_return([{ "pg_is_in_recovery" => "f" }]) + + expect(database.db_read_only?).to be_falsey + end + + it 'detects a read-write database' do + allow(database.model.connection) + .to receive(:execute) + .with('SELECT pg_is_in_recovery()') + .and_return([{ "pg_is_in_recovery" => false }]) + + expect(database.db_read_only?).to be_falsey + end + end + + describe '#db_read_write?' do + it 'detects a read-only database' do + allow(database.model.connection) + .to receive(:execute) + .with('SELECT pg_is_in_recovery()') + .and_return([{ "pg_is_in_recovery" => "t" }]) + + expect(database.db_read_write?).to eq(false) + end + + it 'detects a read-only database' do + allow(database.model.connection) + .to receive(:execute) + .with('SELECT pg_is_in_recovery()') + .and_return([{ "pg_is_in_recovery" => true }]) + + expect(database.db_read_write?).to eq(false) + end + + it 'detects a read-write database' do + allow(database.model.connection) + .to receive(:execute) + .with('SELECT pg_is_in_recovery()') + .and_return([{ "pg_is_in_recovery" => "f" }]) + + expect(database.db_read_write?).to eq(true) + end + + it 'detects a read-write database' do + allow(database.model.connection) + .to receive(:execute) + .with('SELECT pg_is_in_recovery()') + .and_return([{ "pg_is_in_recovery" => false }]) + + expect(database.db_read_write?).to eq(true) + end + end + + describe '#version' do + around do |example| + database.instance_variable_set(:@version, nil) + example.run + database.instance_variable_set(:@version, nil) + end + + context "on postgresql" do + it "extracts the version number" do + allow(database) + .to receive(:database_version) + .and_return("PostgreSQL 9.4.4 on x86_64-apple-darwin14.3.0") + + expect(database.version).to eq '9.4.4' + end + end + + it 'memoizes the result' do + count = ActiveRecord::QueryRecorder + .new { 2.times { database.version } } + .count + + expect(count).to eq(1) + end + end + + describe '#postgresql_minimum_supported_version?' do + it 'returns false when using PostgreSQL 10' do + allow(database).to receive(:version).and_return('10') + + expect(database.postgresql_minimum_supported_version?).to eq(false) + end + + it 'returns false when using PostgreSQL 11' do + allow(database).to receive(:version).and_return('11') + + expect(database.postgresql_minimum_supported_version?).to eq(false) + end + + it 'returns true when using PostgreSQL 12' do + allow(database).to receive(:version).and_return('12') + + expect(database.postgresql_minimum_supported_version?).to eq(true) + end + end + + describe '#cached_column_exists?' do + it 'only retrieves the data from the schema cache' do + database = described_class.new(Project) + queries = ActiveRecord::QueryRecorder.new do + 2.times do + expect(database.cached_column_exists?(:id)).to be_truthy + expect(database.cached_column_exists?(:bogus_column)).to be_falsey + end + end + + expect(queries.count).to eq(0) + end + end + + describe '#cached_table_exists?' do + it 'only retrieves the data from the schema cache' do + dummy = Class.new(ActiveRecord::Base) do + self.table_name = 'bogus_table_name' + end + + queries = ActiveRecord::QueryRecorder.new do + 2.times do + expect(described_class.new(Project).cached_table_exists?).to be_truthy + expect(described_class.new(dummy).cached_table_exists?).to be_falsey + end + end + + expect(queries.count).to eq(0) + end + + it 'returns false when database does not exist' do + database = described_class.new(Project) + + expect(database.model).to receive(:connection) do + raise ActiveRecord::NoDatabaseError, 'broken' + end + + expect(database.cached_table_exists?).to be(false) + end + end + + describe '#exists?' do + it 'returns true if the database exists' do + expect(database.exists?).to be(true) + end + + it "returns false if the database doesn't exist" do + expect(database.model.connection.schema_cache) + .to receive(:database_version) + .and_raise(ActiveRecord::NoDatabaseError) + + expect(database.exists?).to be(false) + end + end + + describe '#system_id' do + it 'returns the PostgreSQL system identifier' do + expect(database.system_id).to be_an_instance_of(Integer) + end + end + + describe '#config' do + it 'returns a HashWithIndifferentAccess' do + expect(database.config) + .to be_an_instance_of(HashWithIndifferentAccess) + end + + it 'returns a default pool size' do + expect(database.config) + .to include(pool: Gitlab::Database.default_pool_size) + end + + it 'does not cache its results' do + a = database.config + b = database.config + + expect(a).not_to equal(b) + end + end +end diff --git a/spec/lib/gitlab/database_spec.rb b/spec/lib/gitlab/database_spec.rb index 1ab08314a3992bdc58057fecbf9cf8c04baf591a..5ec7c338a2adff55cdbb598e316cf7ee96ff0f4d 100644 --- a/spec/lib/gitlab/database_spec.rb +++ b/spec/lib/gitlab/database_spec.rb @@ -15,13 +15,6 @@ end end - describe '.databases' do - it 'stores connections as a HashWithIndifferentAccess' do - expect(described_class.databases.has_key?('main')).to be true - expect(described_class.databases.has_key?(:main)).to be true - end - end - describe '.default_pool_size' do before do allow(Gitlab::Runtime).to receive(:max_threads).and_return(7) @@ -112,18 +105,30 @@ end describe '.check_postgres_version_and_print_warning' do + let(:reflect) { instance_spy(Gitlab::Database::Reflection) } + subject { described_class.check_postgres_version_and_print_warning } + before do + allow(Gitlab::Database::Reflection) + .to receive(:new) + .and_return(reflect) + end + it 'prints a warning if not compliant with minimum postgres version' do - allow(described_class.main).to receive(:postgresql_minimum_supported_version?).and_return(false) + allow(reflect).to receive(:postgresql_minimum_supported_version?).and_return(false) - expect(Kernel).to receive(:warn).with(/You are using PostgreSQL/) + expect(Kernel) + .to receive(:warn) + .with(/You are using PostgreSQL/) + .exactly(Gitlab::Database.database_base_models.length) + .times subject end it 'doesnt print a warning if compliant with minimum postgres version' do - allow(described_class.main).to receive(:postgresql_minimum_supported_version?).and_return(true) + allow(reflect).to receive(:postgresql_minimum_supported_version?).and_return(true) expect(Kernel).not_to receive(:warn).with(/You are using PostgreSQL/) @@ -131,7 +136,7 @@ end it 'doesnt print a warning in Rails runner environment' do - allow(described_class.main).to receive(:postgresql_minimum_supported_version?).and_return(false) + allow(reflect).to receive(:postgresql_minimum_supported_version?).and_return(false) allow(Gitlab::Runtime).to receive(:rails_runner?).and_return(true) expect(Kernel).not_to receive(:warn).with(/You are using PostgreSQL/) @@ -140,13 +145,13 @@ end it 'ignores ActiveRecord errors' do - allow(described_class.main).to receive(:postgresql_minimum_supported_version?).and_raise(ActiveRecord::ActiveRecordError) + allow(reflect).to receive(:postgresql_minimum_supported_version?).and_raise(ActiveRecord::ActiveRecordError) expect { subject }.not_to raise_error end it 'ignores Postgres errors' do - allow(described_class.main).to receive(:postgresql_minimum_supported_version?).and_raise(PG::Error) + allow(reflect).to receive(:postgresql_minimum_supported_version?).and_raise(PG::Error) expect { subject }.not_to raise_error end diff --git a/spec/lib/gitlab/github_import/bulk_importing_spec.rb b/spec/lib/gitlab/github_import/bulk_importing_spec.rb index 6c94973b5a8eb5a478cd9b58649a819f849d5ade..e170496ff7bc0a3c6cbc5ec703b654ec30a982b0 100644 --- a/spec/lib/gitlab/github_import/bulk_importing_spec.rb +++ b/spec/lib/gitlab/github_import/bulk_importing_spec.rb @@ -116,13 +116,13 @@ def object_type value: 5 ) - expect(Gitlab::Database.main) - .to receive(:bulk_insert) + expect(ApplicationRecord) + .to receive(:legacy_bulk_insert) .ordered .with('kittens', rows.first(5)) - expect(Gitlab::Database.main) - .to receive(:bulk_insert) + expect(ApplicationRecord) + .to receive(:legacy_bulk_insert) .ordered .with('kittens', rows.last(5)) diff --git a/spec/lib/gitlab/github_import/importer/diff_note_importer_spec.rb b/spec/lib/gitlab/github_import/importer/diff_note_importer_spec.rb index b671b1268510a401713739f3a0e5a12ef73db7a8..0448ada6bcaa7bad696f7c859cbd7024fb754703 100644 --- a/spec/lib/gitlab/github_import/importer/diff_note_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/diff_note_importer_spec.rb @@ -82,8 +82,8 @@ it 'does not import the note when a foreign key error is raised' do stub_user_finder(project.creator_id, false) - expect(Gitlab::Database.main) - .to receive(:bulk_insert) + expect(ApplicationRecord) + .to receive(:legacy_bulk_insert) .and_raise(ActiveRecord::InvalidForeignKey, 'invalid foreign key') expect { subject.execute } @@ -94,6 +94,8 @@ describe '#execute' do context 'when the merge request no longer exists' do it 'does not import anything' do + expect(ApplicationRecord).not_to receive(:legacy_bulk_insert) + expect { subject.execute } .to not_change(DiffNote, :count) .and not_change(LegacyDiffNote, :count) diff --git a/spec/lib/gitlab/github_import/importer/issue_importer_spec.rb b/spec/lib/gitlab/github_import/importer/issue_importer_spec.rb index 0926000428c4362ed55ba9d15ecbad4a867f030c..4287c32b9475f594483840ee9378cb32c3659fbf 100644 --- a/spec/lib/gitlab/github_import/importer/issue_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/issue_importer_spec.rb @@ -190,8 +190,8 @@ .with(issue.assignees[1]) .and_return(5) - expect(Gitlab::Database.main) - .to receive(:bulk_insert) + expect(ApplicationRecord) + .to receive(:legacy_bulk_insert) .with( IssueAssignee.table_name, [{ issue_id: 1, user_id: 4 }, { issue_id: 1, user_id: 5 }] diff --git a/spec/lib/gitlab/github_import/importer/label_links_importer_spec.rb b/spec/lib/gitlab/github_import/importer/label_links_importer_spec.rb index 241a0fef600f10807c7611a5e471ad42973e810d..e68849755b2affcbc197bde487768d88cc8c61be 100644 --- a/spec/lib/gitlab/github_import/importer/label_links_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/label_links_importer_spec.rb @@ -39,8 +39,8 @@ .and_return(1) freeze_time do - expect(Gitlab::Database.main) - .to receive(:bulk_insert) + expect(ApplicationRecord) + .to receive(:legacy_bulk_insert) .with( LabelLink.table_name, [ @@ -64,8 +64,8 @@ .with('bug') .and_return(nil) - expect(Gitlab::Database.main) - .to receive(:bulk_insert) + expect(ApplicationRecord) + .to receive(:legacy_bulk_insert) .with(LabelLink.table_name, []) importer.create_labels diff --git a/spec/lib/gitlab/github_import/importer/note_importer_spec.rb b/spec/lib/gitlab/github_import/importer/note_importer_spec.rb index 820f46c728607a6f4f6e39184fc2d694f021dfbf..96d8acbd3de9354da76cafd9c312fd46afd194fb 100644 --- a/spec/lib/gitlab/github_import/importer/note_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/note_importer_spec.rb @@ -41,8 +41,8 @@ .with(github_note) .and_return([user.id, true]) - expect(Gitlab::Database.main) - .to receive(:bulk_insert) + expect(ApplicationRecord) + .to receive(:legacy_bulk_insert) .with( Note.table_name, [ @@ -71,8 +71,8 @@ .with(github_note) .and_return([project.creator_id, false]) - expect(Gitlab::Database.main) - .to receive(:bulk_insert) + expect(ApplicationRecord) + .to receive(:legacy_bulk_insert) .with( Note.table_name, [ @@ -115,7 +115,7 @@ context 'when the noteable does not exist' do it 'does not import the note' do - expect(Gitlab::Database.main).not_to receive(:bulk_insert) + expect(ApplicationRecord).not_to receive(:legacy_bulk_insert) importer.execute end @@ -134,8 +134,8 @@ .with(github_note) .and_return([user.id, true]) - expect(Gitlab::Database.main) - .to receive(:bulk_insert) + expect(ApplicationRecord) + .to receive(:legacy_bulk_insert) .and_raise(ActiveRecord::InvalidForeignKey, 'invalid foreign key') expect { importer.execute }.not_to raise_error diff --git a/spec/lib/gitlab/import/database_helpers_spec.rb b/spec/lib/gitlab/import/database_helpers_spec.rb index 079faed25182b3d71dc348083579940e218ec2da..05d1c0ae078858139131cc9d13597e5868c22d64 100644 --- a/spec/lib/gitlab/import/database_helpers_spec.rb +++ b/spec/lib/gitlab/import/database_helpers_spec.rb @@ -16,8 +16,8 @@ let(:project) { create(:project) } it 'returns the ID returned by the query' do - expect(Gitlab::Database.main) - .to receive(:bulk_insert) + expect(ApplicationRecord) + .to receive(:legacy_bulk_insert) .with(Issue.table_name, [attributes], return_ids: true) .and_return([10]) diff --git a/spec/lib/gitlab/metrics/samplers/database_sampler_spec.rb b/spec/lib/gitlab/metrics/samplers/database_sampler_spec.rb index 7dda10ab41dfea1868e1930e322ab87ec3940ff4..e97a4fdddcb40da0f8d91814bac2fe7ff171c11b 100644 --- a/spec/lib/gitlab/metrics/samplers/database_sampler_spec.rb +++ b/spec/lib/gitlab/metrics/samplers/database_sampler_spec.rb @@ -18,8 +18,8 @@ let(:labels) do { class: 'ActiveRecord::Base', - host: Gitlab::Database.main.config['host'], - port: Gitlab::Database.main.config['port'] + host: ApplicationRecord.database.config['host'], + port: ApplicationRecord.database.config['port'] } end diff --git a/spec/lib/gitlab/usage/metrics/instrumentations/generic_metric_spec.rb b/spec/lib/gitlab/usage/metrics/instrumentations/generic_metric_spec.rb index 158be34d39c47a3a547d0dbce2d51c6b609894f2..c8cb1bb437361afeaede826ab1ef097c5f07d803 100644 --- a/spec/lib/gitlab/usage/metrics/instrumentations/generic_metric_spec.rb +++ b/spec/lib/gitlab/usage/metrics/instrumentations/generic_metric_spec.rb @@ -7,18 +7,18 @@ subject do Class.new(described_class) do fallback(custom_fallback) - value { Gitlab::Database.main.version } + value { ApplicationRecord.database.version } end.new(time_frame: 'none') end describe '#value' do it 'gives the correct value' do - expect(subject.value).to eq(Gitlab::Database.main.version) + expect(subject.value).to eq(ApplicationRecord.database.version) end context 'when raising an exception' do it 'return the custom fallback' do - expect(Gitlab::Database.main).to receive(:version).and_raise('Error') + expect(ApplicationRecord.database).to receive(:version).and_raise('Error') expect(subject.value).to eq(custom_fallback) end end @@ -28,18 +28,18 @@ context 'with default fallback' do subject do Class.new(described_class) do - value { Gitlab::Database.main.version } + value { ApplicationRecord.database.version } end.new(time_frame: 'none') end describe '#value' do it 'gives the correct value' do - expect(subject.value).to eq(Gitlab::Database.main.version ) + expect(subject.value).to eq(ApplicationRecord.database.version ) end context 'when raising an exception' do it 'return the default fallback' do - expect(Gitlab::Database.main).to receive(:version).and_raise('Error') + expect(ApplicationRecord.database).to receive(:version).and_raise('Error') expect(subject.value).to eq(described_class::FALLBACK) end end diff --git a/spec/lib/gitlab/usage_data_spec.rb b/spec/lib/gitlab/usage_data_spec.rb index 7cbf439affd3a36f745560986891af48fb2f5a30..cf544c07195063fa9b1fdee039e406c2163c8e7a 100644 --- a/spec/lib/gitlab/usage_data_spec.rb +++ b/spec/lib/gitlab/usage_data_spec.rb @@ -978,9 +978,9 @@ def omniauth_providers expect(subject[:gitlab_pages][:enabled]).to eq(Gitlab.config.pages.enabled) expect(subject[:gitlab_pages][:version]).to eq(Gitlab::Pages::VERSION) expect(subject[:git][:version]).to eq(Gitlab::Git.version) - expect(subject[:database][:adapter]).to eq(Gitlab::Database.main.adapter_name) - expect(subject[:database][:version]).to eq(Gitlab::Database.main.version) - expect(subject[:database][:pg_system_id]).to eq(Gitlab::Database.main.system_id) + expect(subject[:database][:adapter]).to eq(ApplicationRecord.database.adapter_name) + expect(subject[:database][:version]).to eq(ApplicationRecord.database.version) + expect(subject[:database][:pg_system_id]).to eq(ApplicationRecord.database.system_id) expect(subject[:mail][:smtp_server]).to eq(ActionMailer::Base.smtp_settings[:address]) expect(subject[:gitaly][:version]).to be_present expect(subject[:gitaly][:servers]).to be >= 1 diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index 35f8ed3f223db4341397033bf6a7bff654a88b18..b7de8ca4337b83988bd37cbf912eb6241fadca94 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -3806,7 +3806,7 @@ def run_job_without_exception it 'ensures that it is not run in database transaction' do expect(job.pipeline.persistent_ref).to receive(:create) do - expect(Gitlab::Database.main).not_to be_inside_transaction + expect(ApplicationRecord).not_to be_inside_transaction end run_job_without_exception diff --git a/spec/models/concerns/bulk_insert_safe_spec.rb b/spec/models/concerns/bulk_insert_safe_spec.rb index f5c70abf5e980a1219a9eeaa63e814663d9437b2..e6b197f34cae239f5c0546e7f02072da9bc379bb 100644 --- a/spec/models/concerns/bulk_insert_safe_spec.rb +++ b/spec/models/concerns/bulk_insert_safe_spec.rb @@ -182,7 +182,7 @@ def self.invalid_list(count) context 'with returns option set' do let(:items) { bulk_insert_item_class.valid_list(1) } - subject(:bulk_insert) { bulk_insert_item_class.bulk_insert!(items, returns: returns) } + subject(:legacy_bulk_insert) { bulk_insert_item_class.bulk_insert!(items, returns: returns) } context 'when is set to :ids' do let(:returns) { :ids } diff --git a/spec/models/concerns/database_reflection_spec.rb b/spec/models/concerns/database_reflection_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..4111f29ea8d73a5b54ca123babf2d8f166262d51 --- /dev/null +++ b/spec/models/concerns/database_reflection_spec.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe DatabaseReflection do + describe '.reflect' do + it 'returns a Reflection instance' do + expect(User.database).to be_an_instance_of(Gitlab::Database::Reflection) + end + + it 'memoizes the result' do + instance1 = User.database + instance2 = User.database + + expect(instance1).to equal(instance2) + end + end +end diff --git a/spec/models/concerns/legacy_bulk_insert_spec.rb b/spec/models/concerns/legacy_bulk_insert_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..0c6f84f391bfb9502c3cf9a63b15a1d0f9c40ef8 --- /dev/null +++ b/spec/models/concerns/legacy_bulk_insert_spec.rb @@ -0,0 +1,103 @@ +# frozen_string_literal: true + +require 'spec_helper' + +# rubocop: disable Gitlab/BulkInsert +RSpec.describe LegacyBulkInsert do + let(:model) { ApplicationRecord } + + describe '#bulk_insert' do + before do + allow(model).to receive(:connection).and_return(dummy_connection) + allow(dummy_connection).to receive(:quote_column_name, &:itself) + allow(dummy_connection).to receive(:quote, &:itself) + allow(dummy_connection).to receive(:execute) + end + + let(:dummy_connection) { double(:connection) } + + let(:rows) do + [ + { a: 1, b: 2, c: 3 }, + { c: 6, a: 4, b: 5 } + ] + end + + it 'does nothing with empty rows' do + expect(dummy_connection).not_to receive(:execute) + + model.legacy_bulk_insert('test', []) + end + + it 'uses the ordering from the first row' do + expect(dummy_connection).to receive(:execute) do |sql| + expect(sql).to include('(1, 2, 3)') + expect(sql).to include('(4, 5, 6)') + end + + model.legacy_bulk_insert('test', rows) + end + + it 'quotes column names' do + expect(dummy_connection).to receive(:quote_column_name).with(:a) + expect(dummy_connection).to receive(:quote_column_name).with(:b) + expect(dummy_connection).to receive(:quote_column_name).with(:c) + + model.legacy_bulk_insert('test', rows) + end + + it 'quotes values' do + 1.upto(6) do |i| + expect(dummy_connection).to receive(:quote).with(i) + end + + model.legacy_bulk_insert('test', rows) + end + + it 'does not quote values of a column in the disable_quote option' do + [1, 2, 4, 5].each do |i| + expect(dummy_connection).to receive(:quote).with(i) + end + + model.legacy_bulk_insert('test', rows, disable_quote: :c) + end + + it 'does not quote values of columns in the disable_quote option' do + [2, 5].each do |i| + expect(dummy_connection).to receive(:quote).with(i) + end + + model.legacy_bulk_insert('test', rows, disable_quote: [:a, :c]) + end + + it 'handles non-UTF-8 data' do + expect { model.legacy_bulk_insert('test', [{ a: "\255" }]) }.not_to raise_error + end + + context 'when using PostgreSQL' do + it 'allows the returning of the IDs of the inserted rows' do + result = double(:result, values: [['10']]) + + expect(dummy_connection) + .to receive(:execute) + .with(/RETURNING id/) + .and_return(result) + + ids = model + .legacy_bulk_insert('test', [{ number: 10 }], return_ids: true) + + expect(ids).to eq([10]) + end + + it 'allows setting the upsert to do nothing' do + expect(dummy_connection) + .to receive(:execute) + .with(/ON CONFLICT DO NOTHING/) + + model + .legacy_bulk_insert('test', [{ number: 10 }], on_conflict: :do_nothing) + end + end + end +end +# rubocop: enable Gitlab/BulkInsert diff --git a/spec/models/concerns/sha256_attribute_spec.rb b/spec/models/concerns/sha256_attribute_spec.rb index c247865d77f042342caa8223e0155d95fa06e040..02947325bf4208ed08cfc5e54b9f252d3b6ad331 100644 --- a/spec/models/concerns/sha256_attribute_spec.rb +++ b/spec/models/concerns/sha256_attribute_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe Sha256Attribute do - let(:model) { Class.new { include Sha256Attribute } } + let(:model) { Class.new(ApplicationRecord) { include Sha256Attribute } } before do columns = [ diff --git a/spec/models/concerns/sha_attribute_spec.rb b/spec/models/concerns/sha_attribute_spec.rb index 3846dd9c23195ce2712041e0ee8e9b2979610098..220eadfab922b4f531b853f7e1eff201591bb824 100644 --- a/spec/models/concerns/sha_attribute_spec.rb +++ b/spec/models/concerns/sha_attribute_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe ShaAttribute do - let(:model) { Class.new { include ShaAttribute } } + let(:model) { Class.new(ApplicationRecord) { include ShaAttribute } } before do columns = [ diff --git a/spec/models/concerns/x509_serial_number_attribute_spec.rb b/spec/models/concerns/x509_serial_number_attribute_spec.rb index 88550823748daa793a5bd2121acfdefc1cdcf9a7..723e2ad07b67e4ea5cd729d3c2dbddebdb63b1fe 100644 --- a/spec/models/concerns/x509_serial_number_attribute_spec.rb +++ b/spec/models/concerns/x509_serial_number_attribute_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' RSpec.describe X509SerialNumberAttribute do - let(:model) { Class.new { include X509SerialNumberAttribute } } + let(:model) { Class.new(ApplicationRecord) { include X509SerialNumberAttribute } } before do columns = [ diff --git a/spec/models/merge_request_diff_commit_spec.rb b/spec/models/merge_request_diff_commit_spec.rb index e7943af04357903443d62f0f5ba7b31f94f0c294..25e5e40feb7021a37372e479fef75892da79e3af 100644 --- a/spec/models/merge_request_diff_commit_spec.rb +++ b/spec/models/merge_request_diff_commit_spec.rb @@ -71,7 +71,7 @@ subject { described_class.create_bulk(merge_request_diff_id, commits) } it 'inserts the commits into the database en masse' do - expect(Gitlab::Database.main).to receive(:bulk_insert) + expect(ApplicationRecord).to receive(:legacy_bulk_insert) .with(described_class.table_name, rows) subject @@ -114,7 +114,7 @@ end it 'uses a sanitized date' do - expect(Gitlab::Database.main).to receive(:bulk_insert) + expect(ApplicationRecord).to receive(:legacy_bulk_insert) .with(described_class.table_name, rows) subject diff --git a/spec/models/merge_request_diff_spec.rb b/spec/models/merge_request_diff_spec.rb index a96325bf0de38a708c64396285a112ddf5e3d128..afe7251f59aa17513c193f6b2bcc3065af6045b2 100644 --- a/spec/models/merge_request_diff_spec.rb +++ b/spec/models/merge_request_diff_spec.rb @@ -240,8 +240,8 @@ stub_external_diffs_setting(enabled: true) expect(diff).not_to receive(:save!) - expect(Gitlab::Database.main) - .to receive(:bulk_insert) + expect(ApplicationRecord) + .to receive(:legacy_bulk_insert) .with('merge_request_diff_files', anything) .and_raise(ActiveRecord::Rollback) diff --git a/spec/rubocop/cop/gitlab/bulk_insert_spec.rb b/spec/rubocop/cop/gitlab/bulk_insert_spec.rb index bbc8f381d01a8421292c9b4270a8e74402ce64dc..7cd003d0a70ea1f146bf305152c5b3d3b089bc89 100644 --- a/spec/rubocop/cop/gitlab/bulk_insert_spec.rb +++ b/spec/rubocop/cop/gitlab/bulk_insert_spec.rb @@ -6,17 +6,17 @@ RSpec.describe RuboCop::Cop::Gitlab::BulkInsert do subject(:cop) { described_class.new } - it 'flags the use of Gitlab::Database.main.bulk_insert' do + it 'flags the use of ApplicationRecord.legacy_bulk_insert' do expect_offense(<<~SOURCE) - Gitlab::Database.main.bulk_insert('merge_request_diff_files', rows) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Use the `BulkInsertSafe` concern, [...] + ApplicationRecord.legacy_bulk_insert('merge_request_diff_files', rows) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Use the `BulkInsertSafe` concern, [...] SOURCE end - it 'flags the use of ::Gitlab::Database.main.bulk_insert' do + it 'flags the use of ::ApplicationRecord.legacy_bulk_insert' do expect_offense(<<~SOURCE) - ::Gitlab::Database.main.bulk_insert('merge_request_diff_files', rows) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Use the `BulkInsertSafe` concern, [...] + ::ApplicationRecord.legacy_bulk_insert('merge_request_diff_files', rows) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Use the `BulkInsertSafe` concern, [...] SOURCE end end diff --git a/spec/services/packages/create_dependency_service_spec.rb b/spec/services/packages/create_dependency_service_spec.rb index 261c6b395d5dc690b17bc7b5c9be04a6c53760db..55414ea68fe3cc0270da42f82316bdd25d40dac3 100644 --- a/spec/services/packages/create_dependency_service_spec.rb +++ b/spec/services/packages/create_dependency_service_spec.rb @@ -58,9 +58,9 @@ let_it_be(:rows) { [{ name: 'express', version_pattern: '^4.16.4' }] } it 'creates dependences and links' do - original_bulk_insert = ::Gitlab::Database.main.method(:bulk_insert) - expect(::Gitlab::Database.main) - .to receive(:bulk_insert) do |table, rows, return_ids: false, disable_quote: [], on_conflict: nil| + original_bulk_insert = ::ApplicationRecord.method(:legacy_bulk_insert) + expect(::ApplicationRecord) + .to receive(:legacy_bulk_insert) do |table, rows, return_ids: false, disable_quote: [], on_conflict: nil| call_count = table == Packages::Dependency.table_name ? 2 : 1 call_count.times { original_bulk_insert.call(table, rows, return_ids: return_ids, disable_quote: disable_quote, on_conflict: on_conflict) } end.twice diff --git a/spec/services/packages/update_tags_service_spec.rb b/spec/services/packages/update_tags_service_spec.rb index 6e67489fec95268e4f176badc3d67874bb32e76e..c4256699c9466963289ddda52b60c0d6af7be3a6 100644 --- a/spec/services/packages/update_tags_service_spec.rb +++ b/spec/services/packages/update_tags_service_spec.rb @@ -50,7 +50,7 @@ it 'is a no op' do expect(package).not_to receive(:tags) - expect(::Gitlab::Database.main).not_to receive(:bulk_insert) + expect(::ApplicationRecord).not_to receive(:legacy_bulk_insert) subject end diff --git a/spec/services/resource_events/change_labels_service_spec.rb b/spec/services/resource_events/change_labels_service_spec.rb index b987e3204adc2a4239c5478aedfbc75203cd6d5e..c2c0a4c21264edd2c81e3b089570128dbe68c5f5 100644 --- a/spec/services/resource_events/change_labels_service_spec.rb +++ b/spec/services/resource_events/change_labels_service_spec.rb @@ -54,7 +54,7 @@ def expect_label_event(event, label, action) let(:removed) { [labels[1]] } it 'creates all label events in a single query' do - expect(Gitlab::Database.main).to receive(:bulk_insert).once.and_call_original + expect(ApplicationRecord).to receive(:legacy_bulk_insert).once.and_call_original expect { subject }.to change { resource.resource_label_events.count }.from(0).to(2) end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 47aea2c1c374605f76ca8bdacceee2e9e71cc90e..27884617df870787b0c3989503886653f55081af 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -238,7 +238,7 @@ # We can't use an `around` hook here because the wrapping transaction # is not yet opened at the time that is triggered config.prepend_before do - Gitlab::Database.main.set_open_transactions_baseline + ApplicationRecord.set_open_transactions_baseline end config.append_before do @@ -246,7 +246,7 @@ end config.append_after do - Gitlab::Database.main.reset_open_transactions_baseline + ApplicationRecord.reset_open_transactions_baseline end config.before do |example| diff --git a/spec/support/shared_examples/lib/gitlab/database/cte_materialized_shared_examples.rb b/spec/support/shared_examples/lib/gitlab/database/cte_materialized_shared_examples.rb index a617342ff8c103a1f7260891c051d3e273871149..df79572387400e14562d8a5b0c35a1322c5f3cef 100644 --- a/spec/support/shared_examples/lib/gitlab/database/cte_materialized_shared_examples.rb +++ b/spec/support/shared_examples/lib/gitlab/database/cte_materialized_shared_examples.rb @@ -11,7 +11,7 @@ context 'when PG version is <12' do it 'does not add MATERIALIZE keyword' do - allow(Gitlab::Database.main).to receive(:version).and_return('11.1') + allow(ApplicationRecord.database).to receive(:version).and_return('11.1') expect(query).to include(expected_query_block_without_materialized) end @@ -19,14 +19,14 @@ context 'when PG version is >=12' do it 'adds MATERIALIZE keyword' do - allow(Gitlab::Database.main).to receive(:version).and_return('12.1') + allow(ApplicationRecord.database).to receive(:version).and_return('12.1') expect(query).to include(expected_query_block_with_materialized) end context 'when version is higher than 12' do it 'adds MATERIALIZE keyword' do - allow(Gitlab::Database.main).to receive(:version).and_return('15.1') + allow(ApplicationRecord.database).to receive(:version).and_return('15.1') expect(query).to include(expected_query_block_with_materialized) end diff --git a/spec/support/test_reports/test_reports_helper.rb b/spec/support/test_reports/test_reports_helper.rb index 18b40a20cf191c32d28ac5f84619f59049b49862..854830629580bebb0580a790d118c075adc11496 100644 --- a/spec/support/test_reports/test_reports_helper.rb +++ b/spec/support/test_reports/test_reports_helper.rb @@ -95,9 +95,9 @@ def sample_java_failed_message <<-EOF.strip_heredoc junit.framework.AssertionFailedError: expected:<1> but was:<3> at CalculatorTest.subtractExpression(Unknown Source) - at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.base/jdk.internal.database.NativeMethodAccessorImpl.invoke0(Native Method) + at java.base/jdk.internal.database.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at java.base/jdk.internal.database.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) EOF end end diff --git a/spec/tasks/gitlab/db_rake_spec.rb b/spec/tasks/gitlab/db_rake_spec.rb index d051a5536e9b0e16c75879520dbea1432e752b30..38392f77307bbddf9a9e032654e32488f0c42b83 100644 --- a/spec/tasks/gitlab/db_rake_spec.rb +++ b/spec/tasks/gitlab/db_rake_spec.rb @@ -201,7 +201,7 @@ describe 'reindex' do let(:reindex) { double('reindex') } let(:indexes) { double('indexes') } - let(:databases) { Gitlab::Database.databases } + let(:databases) { Gitlab::Database.database_base_models } let(:databases_count) { databases.count } it 'cleans up any leftover indexes' do @@ -250,7 +250,7 @@ end it 'defaults to main database' do - expect(Gitlab::Database::SharedModel).to receive(:using_connection).with(Gitlab::Database.main.scope.connection).and_call_original + expect(Gitlab::Database::SharedModel).to receive(:using_connection).with(ActiveRecord::Base.connection).and_call_original expect do run_rake_task('gitlab:db:enqueue_reindexing_action', "[#{index_name}]") diff --git a/spec/tasks/gitlab/storage_rake_spec.rb b/spec/tasks/gitlab/storage_rake_spec.rb index 570f67c8bb7a6b65d7c22e5ca525df6724c40894..38a031178aedd881b4f018e160db6b84b244b143 100644 --- a/spec/tasks/gitlab/storage_rake_spec.rb +++ b/spec/tasks/gitlab/storage_rake_spec.rb @@ -90,7 +90,7 @@ shared_examples 'wait until database is ready' do it 'checks if the database is ready once' do - expect(Gitlab::Database.main).to receive(:exists?).once + expect(ApplicationRecord.database).to receive(:exists?).once run_rake_task(task) end @@ -102,7 +102,7 @@ end it 'tries for 3 times, polling every 0.1 seconds' do - expect(Gitlab::Database.main).to receive(:exists?).exactly(3).times.and_return(false) + expect(ApplicationRecord.database).to receive(:exists?).exactly(3).times.and_return(false) run_rake_task(task) end