diff --git a/Gemfile.lock b/Gemfile.lock
index 803739ee9b7993e23157e5e194e73899e2619dcb..d5b8c648d4d216102655f2a889ee0c628ba0a3cf 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -27,6 +27,7 @@ PATH
remote: gems/gitlab-active-context
specs:
gitlab-active-context (0.0.1)
+ activerecord
activesupport
connection_pool
elasticsearch
diff --git a/Gemfile.next.lock b/Gemfile.next.lock
index f3578090d7d432156ad4a0450e043676f598af9e..fb51de33921dfd4e6413617e8512cd4436e19c39 100644
--- a/Gemfile.next.lock
+++ b/Gemfile.next.lock
@@ -27,6 +27,7 @@ PATH
remote: gems/gitlab-active-context
specs:
gitlab-active-context (0.0.1)
+ activerecord
activesupport
connection_pool
elasticsearch
diff --git a/ee/db/active_context/migrate/.gitkeep b/ee/db/active_context/migrate/.gitkeep
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/gems/gitlab-active-context/.rubocop.yml b/gems/gitlab-active-context/.rubocop.yml
index 29f1c5a30209a2d49e1cf89e3107fb7eb6fd887e..65518b19e7d722de0827443777c6209c2da11b4a 100644
--- a/gems/gitlab-active-context/.rubocop.yml
+++ b/gems/gitlab-active-context/.rubocop.yml
@@ -7,6 +7,9 @@ Gemfile/MissingFeatureCategory:
Search/NamespacedClass:
Enabled: false
+RSpec/AnyInstanceOf:
+ Enabled: false
+
RSpec/MultipleMemoizedHelpers:
Max: 25
@@ -14,6 +17,10 @@ RSpec/VerifiedDoubles:
Exclude:
- 'spec/lib/active_context/tracker_spec.rb'
+Naming/ClassAndModuleCamelCase:
+ AllowedNames:
+ - V1_0
+
RSpec/VerifiedDoubleReference:
Exclude:
- 'spec/lib/active_context/bulk_process_queue_spec.rb'
diff --git a/gems/gitlab-active-context/Gemfile.lock b/gems/gitlab-active-context/Gemfile.lock
index 6f5a395e2bfb623d607e3ae3428f954ffb08b98e..3b01b42833730847ca8fcc625cc4b9d4aebeff6b 100644
--- a/gems/gitlab-active-context/Gemfile.lock
+++ b/gems/gitlab-active-context/Gemfile.lock
@@ -2,6 +2,7 @@ PATH
remote: .
specs:
gitlab-active-context (0.0.1)
+ activerecord
activesupport
connection_pool
elasticsearch
@@ -28,6 +29,12 @@ GEM
erubi (~> 1.11)
rails-dom-testing (~> 2.2)
rails-html-sanitizer (~> 1.6)
+ activemodel (8.0.1)
+ activesupport (= 8.0.1)
+ activerecord (8.0.1)
+ activemodel (= 8.0.1)
+ activesupport (= 8.0.1)
+ timeout (>= 0.4.0)
activesupport (8.0.1)
base64
benchmark (>= 0.3)
@@ -222,6 +229,7 @@ GEM
securerandom (0.4.0)
stringio (3.1.2)
thor (1.3.2)
+ timeout (0.4.3)
tzinfo (2.0.6)
concurrent-ruby (~> 1.0)
unicode-display_width (2.6.0)
diff --git a/gems/gitlab-active-context/gitlab-active-context.gemspec b/gems/gitlab-active-context/gitlab-active-context.gemspec
index 9ff6c3b07ee703535d329695a070d3c076246c50..f19f1452f5580673684db74ab83379839debbddf 100644
--- a/gems/gitlab-active-context/gitlab-active-context.gemspec
+++ b/gems/gitlab-active-context/gitlab-active-context.gemspec
@@ -19,6 +19,7 @@ Gem::Specification.new do |spec|
spec.files = Dir['lib/**/*.rb']
spec.require_paths = ["lib"]
+ spec.add_dependency 'activerecord'
spec.add_dependency 'activesupport'
spec.add_dependency 'connection_pool'
spec.add_dependency 'elasticsearch'
diff --git a/gems/gitlab-active-context/lib/active_context.rb b/gems/gitlab-active-context/lib/active_context.rb
index a7c8cbe860144dc1a536483c10396a0c2a034f2c..48d0bf24a9c1a3c942f35111b62c00f753870ff3 100644
--- a/gems/gitlab-active-context/lib/active_context.rb
+++ b/gems/gitlab-active-context/lib/active_context.rb
@@ -5,6 +5,7 @@
require 'connection_pool'
require 'pg'
require 'zeitwerk'
+require 'active_record'
loader = Zeitwerk::Loader.for_gem
loader.setup
diff --git a/gems/gitlab-active-context/lib/active_context/config.rb b/gems/gitlab-active-context/lib/active_context/config.rb
index 604007563944760a94ce804d69608cf1d765d420..b68843f65e11b04f937c4f62977642b4b3e4d9b1 100644
--- a/gems/gitlab-active-context/lib/active_context/config.rb
+++ b/gems/gitlab-active-context/lib/active_context/config.rb
@@ -2,7 +2,7 @@
module ActiveContext
class Config
- Cfg = Struct.new(:enabled, :databases, :logger, :indexing_enabled, :re_enqueue_indexing_workers)
+ Cfg = Struct.new(:enabled, :databases, :logger, :indexing_enabled, :re_enqueue_indexing_workers, :migrations_path)
class << self
def configure(&block)
@@ -21,8 +21,12 @@ def databases
current.databases || {}
end
+ def migrations_path
+ current.migrations_path || Rails.root.join('ee/db/active_context/migrate')
+ end
+
def logger
- current.logger || Logger.new($stdout)
+ current.logger || ::Logger.new($stdout)
end
def indexing_enabled?
diff --git a/gems/gitlab-active-context/lib/active_context/databases/collection_builder.rb b/gems/gitlab-active-context/lib/active_context/databases/collection_builder.rb
new file mode 100644
index 0000000000000000000000000000000000000000..00949de9124696bfc474a37214c129f07a4399cc
--- /dev/null
+++ b/gems/gitlab-active-context/lib/active_context/databases/collection_builder.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module ActiveContext
+ module Databases
+ class CollectionBuilder
+ attr_reader :fields
+
+ def initialize
+ @fields = []
+ end
+
+ def bigint(name, index: false)
+ fields << Field::Bigint.new(name, index: index)
+ end
+
+ def prefix(name)
+ fields << Field::Prefix.new(name, index: true)
+ end
+
+ def vector(name, dimensions:, index: true)
+ fields << Field::Vector.new(name, dimensions: dimensions, index: index)
+ end
+ end
+
+ class Field
+ attr_reader :name, :options
+
+ def initialize(name, **options)
+ @name = name.to_s
+ @options = options
+ end
+
+ class Bigint < Field; end
+ class Prefix < Field; end
+ class Vector < Field; end
+ end
+ end
+end
diff --git a/gems/gitlab-active-context/lib/active_context/databases/concerns/adapter.rb b/gems/gitlab-active-context/lib/active_context/databases/concerns/adapter.rb
index 51ef809e8c138b64cc7ea082115c46f4fbabda62..65aa29deafb2bdee1901bbe7efe0c6938b00bf69 100644
--- a/gems/gitlab-active-context/lib/active_context/databases/concerns/adapter.rb
+++ b/gems/gitlab-active-context/lib/active_context/databases/concerns/adapter.rb
@@ -4,15 +4,20 @@ module ActiveContext
module Databases
module Concerns
module Adapter
- attr_reader :options, :client, :indexer
+ attr_reader :options, :prefix, :client, :indexer, :executor
+
+ DEFAULT_PREFIX = 'gitlab_active_context'
+ DEFAULT_SEPARATOR = '_'
delegate :search, to: :client
delegate :all_refs, :add_ref, :empty?, :bulk, :process_bulk_errors, :reset, to: :indexer
def initialize(options)
@options = options
+ @prefix = options[:prefix] || DEFAULT_PREFIX
@client = client_klass.new(options)
@indexer = indexer_klass.new(options, client)
+ @executor = executor_klass.new(self)
end
def client_klass
@@ -22,6 +27,18 @@ def client_klass
def indexer_klass
raise NotImplementedError
end
+
+ def executor_klass
+ raise NotImplementedError
+ end
+
+ def full_collection_name(name)
+ [prefix, name].compact.join(separator)
+ end
+
+ def separator
+ DEFAULT_SEPARATOR
+ end
end
end
end
diff --git a/gems/gitlab-active-context/lib/active_context/databases/concerns/client.rb b/gems/gitlab-active-context/lib/active_context/databases/concerns/client.rb
index a27ecb45907beb59d6d7935cbb5a765b2a5cdaec..899e13082b919dbbb2463ec04420d167a9f93cef 100644
--- a/gems/gitlab-active-context/lib/active_context/databases/concerns/client.rb
+++ b/gems/gitlab-active-context/lib/active_context/databases/concerns/client.rb
@@ -8,10 +8,6 @@ module Client
attr_reader :options
- def prefix
- options[:prefix] || DEFAULT_PREFIX
- end
-
def search(_)
raise NotImplementedError
end
diff --git a/gems/gitlab-active-context/lib/active_context/databases/concerns/executor.rb b/gems/gitlab-active-context/lib/active_context/databases/concerns/executor.rb
new file mode 100644
index 0000000000000000000000000000000000000000..3ca6974f320734e2ff8a61ce911b04848893fd82
--- /dev/null
+++ b/gems/gitlab-active-context/lib/active_context/databases/concerns/executor.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module ActiveContext
+ module Databases
+ module Concerns
+ module Executor
+ attr_reader :adapter
+
+ def initialize(adapter)
+ @adapter = adapter
+ end
+
+ def create_collection(name, number_of_partitions:, &block)
+ builder = ActiveContext::Databases::CollectionBuilder.new
+ yield(builder) if block
+
+ full_name = adapter.full_collection_name(name)
+ do_create_collection(
+ name: full_name,
+ number_of_partitions: number_of_partitions,
+ fields: builder.fields
+ )
+ end
+
+ private
+
+ def do_create_collection(...)
+ raise NotImplementedError
+ end
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-active-context/lib/active_context/databases/elasticsearch/adapter.rb b/gems/gitlab-active-context/lib/active_context/databases/elasticsearch/adapter.rb
index a75894a347976dac9e4e7b0c0b6e2c1990acb7ee..bb46833f218bcf6522363f95ee88cdb178c81744 100644
--- a/gems/gitlab-active-context/lib/active_context/databases/elasticsearch/adapter.rb
+++ b/gems/gitlab-active-context/lib/active_context/databases/elasticsearch/adapter.rb
@@ -13,6 +13,10 @@ def client_klass
def indexer_klass
ActiveContext::Databases::Elasticsearch::Indexer
end
+
+ def executor_klass
+ ActiveContext::Databases::Elasticsearch::Executor
+ end
end
end
end
diff --git a/gems/gitlab-active-context/lib/active_context/databases/elasticsearch/executor.rb b/gems/gitlab-active-context/lib/active_context/databases/elasticsearch/executor.rb
new file mode 100644
index 0000000000000000000000000000000000000000..49d3d169b64a43d0b6ad81b334ed54b226e94fab
--- /dev/null
+++ b/gems/gitlab-active-context/lib/active_context/databases/elasticsearch/executor.rb
@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module ActiveContext
+ module Databases
+ module Elasticsearch
+ class Executor
+ include ActiveContext::Databases::Concerns::Executor
+
+ private
+
+ def raw_client
+ @raw_client ||= adapter.client.client
+ end
+
+ def do_create_collection(name:, number_of_partitions:, fields:)
+ strategy = PartitionStrategy.new(
+ name: name,
+ number_of_partitions: number_of_partitions
+ )
+
+ # Early return if everything exists
+ return if collection_exists?(strategy)
+
+ # Create missing partitions
+ strategy.each_partition do |partition_name|
+ create_partition(partition_name, fields) unless index_exists?(partition_name)
+ end
+
+ # Create alias if needed
+ create_alias(strategy) unless alias_exists?(strategy.collection_name)
+ end
+
+ def create_partition(name, fields)
+ mappings = {
+ mappings: {
+ properties: build_field_mappings(fields)
+ }
+ }
+ raw_client.indices.create(index: name, body: mappings)
+ end
+
+ def create_alias(strategy)
+ actions = [{
+ add: {
+ indices: strategy.partition_names,
+ alias: strategy.collection_name
+ }
+ }]
+ raw_client.indices.update_aliases(body: { actions: actions })
+ end
+
+ def build_field_mappings(fields)
+ fields.each_with_object({}) do |field, mappings|
+ mappings[field.name] = case field
+ when Field::Bigint
+ { type: 'long' }
+ when Field::Prefix
+ { type: 'keyword' }
+ when Field::Vector
+ {
+ type: 'dense_vector',
+ dims: field.options[:dimensions],
+ index: true,
+ similarity: 'cosine'
+ }
+ else
+ raise ArgumentError, "Unknown field type: #{field.class}"
+ end
+ end
+ end
+
+ def collection_exists?(strategy)
+ return false unless alias_exists?(strategy.collection_name)
+
+ strategy.fully_exists? do |partition_name|
+ index_exists?(partition_name)
+ end
+ end
+
+ def index_exists?(name)
+ raw_client.indices.exists?(index: name)
+ end
+
+ def alias_exists?(name)
+ raw_client.indices.exists_alias?(name: name)
+ end
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-active-context/lib/active_context/databases/opensearch/adapter.rb b/gems/gitlab-active-context/lib/active_context/databases/opensearch/adapter.rb
index a72fac16d38a825a5df640cd5163cdb701ffb106..9a6ed90ef4eaca7042dec1dd2e4f4a44b4726f81 100644
--- a/gems/gitlab-active-context/lib/active_context/databases/opensearch/adapter.rb
+++ b/gems/gitlab-active-context/lib/active_context/databases/opensearch/adapter.rb
@@ -13,6 +13,10 @@ def client_klass
def indexer_klass
ActiveContext::Databases::Opensearch::Indexer
end
+
+ def executor_klass
+ ActiveContext::Databases::Opensearch::Executor
+ end
end
end
end
diff --git a/gems/gitlab-active-context/lib/active_context/databases/opensearch/executor.rb b/gems/gitlab-active-context/lib/active_context/databases/opensearch/executor.rb
new file mode 100644
index 0000000000000000000000000000000000000000..de6fef86241f137082743c6e0a9c3d2d023fd0e7
--- /dev/null
+++ b/gems/gitlab-active-context/lib/active_context/databases/opensearch/executor.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module ActiveContext
+ module Databases
+ module Opensearch
+ class Executor
+ include ActiveContext::Databases::Concerns::Executor
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-active-context/lib/active_context/databases/partition_strategy.rb b/gems/gitlab-active-context/lib/active_context/databases/partition_strategy.rb
new file mode 100644
index 0000000000000000000000000000000000000000..24fba7c29a770994431a5b85b91c46fe1f56e412
--- /dev/null
+++ b/gems/gitlab-active-context/lib/active_context/databases/partition_strategy.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module ActiveContext
+ module Databases
+ class PartitionStrategy
+ def initialize(name:, number_of_partitions:)
+ @name = name
+ @number_of_partitions = number_of_partitions
+ end
+
+ # Returns list of all partition names for this collection
+ def partition_names
+ Array.new(@number_of_partitions) do |i|
+ generate_partition_name(i)
+ end
+ end
+
+ # Returns the collection alias/view name
+ def collection_name
+ @name
+ end
+
+ # Generates a specific partition name given an index
+ def generate_partition_name(index)
+ "#{@name}#{ActiveContext.adapter.separator}#{index}"
+ end
+
+ # Helps executors check if collection is fully set up
+ def fully_exists?(&partition_exists_check)
+ partition_names.all?(&partition_exists_check)
+ end
+
+ # Iterator for operating on partitions
+ def each_partition
+ partition_names.each do |name|
+ yield name
+ end
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-active-context/lib/active_context/databases/postgresql/adapter.rb b/gems/gitlab-active-context/lib/active_context/databases/postgresql/adapter.rb
index 3d71ee59d50ed45bc14b8d3138fee74142b52e85..dcea44fb7cc511639e4d150fbefc9e17d44c6ef2 100644
--- a/gems/gitlab-active-context/lib/active_context/databases/postgresql/adapter.rb
+++ b/gems/gitlab-active-context/lib/active_context/databases/postgresql/adapter.rb
@@ -13,6 +13,10 @@ def client_klass
def indexer_klass
ActiveContext::Databases::Postgresql::Indexer
end
+
+ def executor_klass
+ ActiveContext::Databases::Postgresql::Executor
+ end
end
end
end
diff --git a/gems/gitlab-active-context/lib/active_context/databases/postgresql/client.rb b/gems/gitlab-active-context/lib/active_context/databases/postgresql/client.rb
index 1ab361b20b38acc6da217b544f95837ae2d99b3d..ccc1c7afdfe5645c19629fa22d4b13ff88d937db 100644
--- a/gems/gitlab-active-context/lib/active_context/databases/postgresql/client.rb
+++ b/gems/gitlab-active-context/lib/active_context/databases/postgresql/client.rb
@@ -6,48 +6,126 @@ module Postgresql
class Client
include ActiveContext::Databases::Concerns::Client
+ class << self
+ attr_accessor :default_connection_pool
+ end
+
DEFAULT_POOL_SIZE = 5
- DEFAULT_POOL_TIMEOUT = 5
+ DEFAULT_CONNECT_TIMEOUT = 5
+
+ attr_reader :connection_pool, :options
def initialize(options)
@options = options
- @pool = ConnectionPool.new(
- size: options.fetch(:pool_size, DEFAULT_POOL_SIZE),
- timeout: options.fetch(:pool_timeout, DEFAULT_POOL_TIMEOUT)
- ) do
- PG.connect(connection_params)
- end
+ setup_connection_pool
end
def search(_query)
with_connection do |conn|
- res = conn.exec('SELECT * FROM pg_stat_activity')
+ res = conn.execute('SELECT * FROM pg_stat_activity')
QueryResult.new(res)
end
end
+ # Provides raw PostgreSQL connection
+ def with_raw_connection(&block)
+ handle_connection(raw_connection: true, &block)
+ end
+
+ # Provides Rails-wrapped connection for using ActiveRecord methods
+ def with_connection(&block)
+ handle_connection(raw_connection: false, &block)
+ end
+
+ # Creates an ActiveRecord model for a specific table and yields it within the connection context
+ # @param table_name [String] The name of the table to create a model for
+ # @yield [Class] A dynamically created ActiveRecord model class with the correct connection
+ def with_model_for(table_name)
+ model_class = Class.new(::ActiveRecord::Base) do
+ self.table_name = table_name
+
+ def self.name
+ "ActiveContext::Model::#{table_name.classify}"
+ end
+
+ def self.to_s
+ name
+ end
+ end
+
+ with_connection do |conn|
+ model_class.define_singleton_method(:connection) { conn }
+ yield model_class
+ end
+ end
+
+ # For backward compatibility and simpler queries
+ def ar_model_for(table_name)
+ klass = nil
+ with_model_for(table_name) do |model_class|
+ klass = model_class
+ end
+ klass
+ end
+
private
- def with_connection
- @pool.with do |conn|
- yield(conn)
+ def handle_connection(raw_connection: false)
+ connection_pool.with_connection do |conn|
+ yield(raw_connection ? conn.raw_connection : conn)
+ rescue PG::Error, ::ActiveRecord::StatementInvalid => e
+ handle_error(e)
end
end
- def close
- @pool&.shutdown(&:close)
+ def handle_error(error)
+ ActiveContext::Logger.exception(error, message: 'Database error occurred')
+ raise error
+ end
+
+ def setup_connection_pool
+ model_class = create_connection_model
+ model_class.establish_connection(build_database_config.stringify_keys)
+ @connection_pool = model_class.connection_pool
+ end
+
+ def create_connection_model
+ Class.new(::ActiveRecord::Base) do
+ self.abstract_class = true
+
+ def self.name
+ "ActiveContext::ConnectionPool::#{object_id}"
+ end
+
+ def self.to_s
+ name
+ end
+ end
end
- def connection_params
+ def build_database_config
{
+ adapter: 'postgresql',
host: options[:host],
port: options[:port],
- dbname: options[:database],
- user: options[:username],
+ database: options[:database],
+ username: options[:username],
password: options[:password],
- connect_timeout: options.fetch(:connect_timeout, 5)
+ connect_timeout: options.fetch(:connect_timeout, DEFAULT_CONNECT_TIMEOUT),
+ pool: calculate_pool_size,
+ prepared_statements: false,
+ advisory_locks: false,
+ database_tasks: false # This signals Rails that this is an auxiliary database
}.compact
end
+
+ def calculate_pool_size
+ options[:pool_size] || DEFAULT_POOL_SIZE
+ end
+
+ def close
+ connection_pool&.disconnect!
+ end
end
end
end
diff --git a/gems/gitlab-active-context/lib/active_context/databases/postgresql/executor.rb b/gems/gitlab-active-context/lib/active_context/databases/postgresql/executor.rb
new file mode 100644
index 0000000000000000000000000000000000000000..8ccb40e7a50bc492a4443a897922b51df00b1add
--- /dev/null
+++ b/gems/gitlab-active-context/lib/active_context/databases/postgresql/executor.rb
@@ -0,0 +1,198 @@
+# frozen_string_literal: true
+
+module ActiveContext
+ module Databases
+ module Postgresql
+ class Executor
+ include ActiveContext::Databases::Concerns::Executor
+
+ private
+
+ def do_create_collection(name:, number_of_partitions:, fields:)
+ strategy = PartitionStrategy.new(
+ name: name,
+ number_of_partitions: number_of_partitions
+ )
+
+ return if collection_exists?(strategy)
+
+ # Create parent table if it doesn't exist
+ create_parent_table(strategy.collection_name, fields) unless table_exists?(strategy.collection_name)
+
+ # Create child partition tables
+ strategy.each_partition do |partition_name|
+ next if table_exists?(partition_name)
+
+ create_partition_table(
+ partition_name,
+ strategy.collection_name,
+ get_partition_remainder(partition_name)
+ )
+ end
+
+ # Create indices on fields that need them
+ create_indices(strategy, fields)
+ end
+
+ def create_parent_table(name, fields)
+ fixed_columns, variable_columns = sort_fields_by_size(fields)
+
+ adapter.client.with_connection do |connection|
+ connection.create_table(name, id: false, primary_key: [:id, :partition_id],
+ options: 'PARTITION BY LIST (partition_id)') do |table|
+ # Add partition_id first as it's required for partitioning
+ table.integer :partition_id, null: false
+
+ # Add fixed columns first for better memory alignment
+ fixed_columns.each do |field|
+ add_column_from_field(table, field)
+ end
+
+ # Add id column
+ table.string :id, null: false
+
+ # Add variable width columns last
+ variable_columns.each do |field|
+ add_column_from_field(table, field)
+ end
+ end
+ end
+ end
+
+ def sort_fields_by_size(fields)
+ fixed_columns = []
+ variable_columns = []
+
+ fields.each do |field|
+ case field
+ when Field::Vector
+ # Vector fields have fixed size based on dimensions
+ fixed_columns << [field, field.options[:dimensions] * 4]
+ when Field::Bigint
+ # Bigint is 8 bytes
+ fixed_columns << [field, 8]
+ when Field::Prefix
+ # Text fields are variable width
+ variable_columns << field
+ else
+ raise ArgumentError, "Unknown field type: #{field.class}"
+ end
+ end
+
+ # Sort fixed-size columns by size in descending order for best alignment
+ [fixed_columns.sort_by { |_, size| -size }.map(&:first), variable_columns]
+ end
+
+ def add_column_from_field(table, field)
+ case field
+ when Field::Vector
+ table.column(field.name, "vector(#{field.options[:dimensions]})")
+ when Field::Bigint
+ table.bigint(field.name, **field.options.except(:index))
+ when Field::Prefix
+ table.text(field.name, **field.options.except(:index))
+ else
+ raise ArgumentError, "Unknown field type: #{field.class}"
+ end
+ end
+
+ def add_column_from_definition(table, column_def, _connection)
+ name, type_info = parse_column_definition(column_def)
+
+ if type_info[:type] == :virtual
+ # For vector columns, use raw SQL type
+ table.column(name, type_info[:options][:as])
+ else
+ table.column(name, type_info[:type], **type_info[:options])
+ end
+ end
+
+ def create_partition_table(partition_name, parent_name, partition_id)
+ adapter.client.with_connection do |connection|
+ sql = <<~SQL.squish
+ CREATE TABLE #{connection.quote_table_name(partition_name)}
+ PARTITION OF #{connection.quote_table_name(parent_name)}
+ FOR VALUES IN (#{partition_id});
+ SQL
+
+ connection.execute(sql)
+ end
+ end
+
+ def create_indices(strategy, fields)
+ fields.each do |field|
+ next unless field.options[:index]
+
+ if field.is_a?(Field::Vector)
+ strategy.each_partition do |partition_name|
+ next if index_exists?(partition_name, field)
+
+ create_vector_index(partition_name, field)
+ end
+ else
+ create_standard_index(strategy.collection_name, field)
+ end
+ end
+ end
+
+ def create_standard_index(table_name, field)
+ adapter.client.with_connection do |connection|
+ next if index_exists?(table_name, field)
+
+ connection.add_index(
+ table_name,
+ field.name,
+ name: index_name_for(table_name, field)
+ )
+ end
+ end
+
+ def create_vector_index(table_name, field)
+ adapter.client.with_connection do |connection|
+ next if index_exists?(table_name, field)
+
+ index_name = index_name_for(table_name, field)
+
+ connection.execute(<<~SQL.squish)
+ CREATE INDEX #{connection.quote_column_name(index_name)}
+ ON #{connection.quote_table_name(table_name)}
+ USING hnsw (#{connection.quote_column_name(field.name)} vector_l2_ops)
+ SQL
+ end
+ end
+
+ def index_exists?(table_name, field)
+ adapter.client.with_connection do |connection|
+ index_name = index_name_for(table_name, field)
+ connection.index_exists?(table_name, field.name, name: index_name)
+ end
+ end
+
+ def index_name_for(table_name, field)
+ "#{table_name}_#{field.name}_idx"
+ end
+
+ def collection_exists?(strategy)
+ adapter.client.with_connection do |connection|
+ exists = connection.table_exists?(strategy.collection_name)
+ next false unless exists
+
+ strategy.partition_names.all? do |partition_name|
+ connection.table_exists?(partition_name)
+ end
+ end
+ end
+
+ def table_exists?(name)
+ adapter.client.with_connection do |connection|
+ connection.table_exists?(name)
+ end
+ end
+
+ def get_partition_remainder(partition_name)
+ partition_name.split('_').last.to_i
+ end
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-active-context/lib/active_context/logger.rb b/gems/gitlab-active-context/lib/active_context/logger.rb
new file mode 100644
index 0000000000000000000000000000000000000000..72094562e14575cab0ae1920c8027d1175aaf5d0
--- /dev/null
+++ b/gems/gitlab-active-context/lib/active_context/logger.rb
@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module ActiveContext
+ class Logger
+ ANONYMOUS = '<Anonymous>'
+
+ class << self
+ def debug(**kwargs)
+ log(:debug, **kwargs)
+ end
+
+ def info(**kwargs)
+ log(:info, **kwargs)
+ end
+
+ def warn(**kwargs)
+ log(:warn, **kwargs)
+ end
+
+ def error(**kwargs)
+ log(:error, **kwargs)
+ end
+
+ def fatal(**kwargs)
+ log(:fatal, **kwargs)
+ end
+
+ def exception(exception, **kwargs)
+ payload = {
+ exception_class: exception.class.name,
+ exception_message: exception.message,
+ exception_backtrace: exception.backtrace
+ }.merge(kwargs)
+
+ error(**payload)
+ end
+
+ private
+
+ def log(severity, **kwargs)
+ logger = ActiveContext::Config.logger
+
+ return unless logger
+
+ payload = build_structured_payload(**kwargs)
+ case severity
+ when :debug then logger.debug(payload)
+ when :info then logger.info(payload)
+ when :warn then logger.warn(payload)
+ when :error then logger.error(payload)
+ when :fatal then logger.fatal(payload)
+ end
+ end
+
+ def build_structured_payload(**params)
+ { class: self.class.name || ANONYMOUS }.merge(params).stringify_keys
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-active-context/lib/active_context/migration.rb b/gems/gitlab-active-context/lib/active_context/migration.rb
new file mode 100644
index 0000000000000000000000000000000000000000..69f24c5e769a6dae4c406c7b307a098e65293fb2
--- /dev/null
+++ b/gems/gitlab-active-context/lib/active_context/migration.rb
@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module ActiveContext
+ class Migration
+ class V1_0
+ class << self
+ def milestone(version)
+ @milestone = version
+ end
+
+ def milestone_version
+ @milestone
+ end
+ end
+
+ def initialize
+ @operations = {}
+ end
+
+ def migrate!
+ raise NotImplementedError, "#{self.class.name} must implement #migrate!"
+ end
+
+ def create_collection(name, **options, &block)
+ operation = @operations[:"create_#{name}"] ||= OperationResult.new("create_#{name}")
+
+ # Only execute if not already completed
+ unless operation.completed?
+ ActiveContext.adapter.executor.create_collection(name, **options, &block)
+ operation.complete!
+ end
+
+ operation.completed?
+ end
+ end
+
+ def self.[](version)
+ version = version.to_s
+ name = "V#{version.tr('.', '_')}"
+
+ raise ArgumentError, "Unknown migration version: #{version}" unless const_defined?(name, false)
+
+ const_get(name, false)
+ end
+
+ def self.current_version
+ 1.0
+ end
+ end
+end
diff --git a/gems/gitlab-active-context/lib/active_context/migration/dictionary.rb b/gems/gitlab-active-context/lib/active_context/migration/dictionary.rb
new file mode 100644
index 0000000000000000000000000000000000000000..9b8ddd05248232ff0326b2d94760214beb64dbf7
--- /dev/null
+++ b/gems/gitlab-active-context/lib/active_context/migration/dictionary.rb
@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module ActiveContext
+ class Migration
+ class Dictionary
+ Error = Class.new(StandardError)
+ DuplicateVersionError = Class.new(Error)
+ InvalidMigrationNameError = Class.new(Error)
+
+ class << self
+ def migrations_path
+ Array(ActiveContext::Config.migrations_path)
+ end
+
+ def instance
+ @instance ||= new
+ end
+
+ delegate :migrations, to: :instance
+ end
+
+ def initialize
+ @migrations = {}
+ load_migrations
+ end
+
+ # Returns all migrations sorted by version
+ def migrations
+ @migrations.sort_by { |version, _| version }.map(&:last)
+ end
+
+ # Find a specific migration by version
+ def find_by_version(version)
+ @migrations[version.to_s]
+ end
+
+ private
+
+ def load_migrations
+ migration_files.each do |file|
+ version, name = parse_migration_file(file)
+ version_constant = :"V#{version}"
+
+ # Only define the module if it doesn't exist
+ if ActiveContext::Migration.const_defined?(version_constant)
+ migration_module = ActiveContext::Migration.const_get(version_constant, false)
+ else
+ migration_module = Module.new
+ ActiveContext::Migration.const_set(version_constant, migration_module)
+
+ # Evaluate the migration file content within the namespace
+ migration_content = File.read(file)
+ migration_module.module_eval(migration_content)
+ end
+
+ klass_name = name.camelize
+ begin
+ # Look up the class within our namespace
+ klass = migration_module.const_get(klass_name, false)
+ rescue NameError
+ raise InvalidMigrationNameError, "Could not find migration class '#{klass_name}' in #{file}"
+ end
+
+ if @migrations.key?(version)
+ raise DuplicateVersionError, "Multiple migrations have the version number #{version}"
+ end
+
+ @migrations[version] = klass
+ end
+ end
+
+ def migration_files
+ self.class.migrations_path.flat_map do |path|
+ Dir[File.join(path, '*.rb')]
+ end
+ end
+
+ def parse_migration_file(filename)
+ basename = File.basename(filename, '.rb')
+
+ if basename =~ /\A([0-9]{14})_(.+)\z/
+ version = ::Regexp.last_match(1)
+ name = ::Regexp.last_match(2)
+ [version, name]
+ else
+ raise InvalidMigrationNameError,
+ "Invalid migration file name format: #{basename}. Expected format: YYYYMMDDHHMMSS_name.rb"
+ end
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-active-context/lib/active_context/migration/operation_result.rb b/gems/gitlab-active-context/lib/active_context/migration/operation_result.rb
new file mode 100644
index 0000000000000000000000000000000000000000..823086ce62d1a0ad925304d3ae3aef763f04c1a0
--- /dev/null
+++ b/gems/gitlab-active-context/lib/active_context/migration/operation_result.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module ActiveContext
+ class Migration
+ class OperationResult
+ attr_reader :operation_name, :completed
+ alias_method :completed?, :completed
+
+ def initialize(operation_name)
+ @operation_name = operation_name
+ @completed = false
+ end
+
+ def complete!
+ @completed = true
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-active-context/spec/lib/active_context/databases/elasticsearch/adapter_spec.rb b/gems/gitlab-active-context/spec/lib/active_context/databases/elasticsearch/adapter_spec.rb
index 7dd9660b36f3160a6ad08b46ebfa0246bb1d11a6..7d183a69d58fb5a7c748689e6171db43964b1170 100644
--- a/gems/gitlab-active-context/spec/lib/active_context/databases/elasticsearch/adapter_spec.rb
+++ b/gems/gitlab-active-context/spec/lib/active_context/databases/elasticsearch/adapter_spec.rb
@@ -11,4 +11,15 @@
adapter.search(query)
end
+
+ describe '#prefix' do
+ it 'returns default prefix when not specified' do
+ expect(adapter.prefix).to eq('gitlab_active_context')
+ end
+
+ it 'returns configured prefix' do
+ adapter = described_class.new(options.merge(prefix: 'custom'))
+ expect(adapter.prefix).to eq('custom')
+ end
+ end
end
diff --git a/gems/gitlab-active-context/spec/lib/active_context/databases/elasticsearch/client_spec.rb b/gems/gitlab-active-context/spec/lib/active_context/databases/elasticsearch/client_spec.rb
index bf313814ab95c8441ead97062045c776b5d1aafd..52d31479046c03d784e0e24fb5de2b31e06da000 100644
--- a/gems/gitlab-active-context/spec/lib/active_context/databases/elasticsearch/client_spec.rb
+++ b/gems/gitlab-active-context/spec/lib/active_context/databases/elasticsearch/client_spec.rb
@@ -52,15 +52,4 @@
)
end
end
-
- describe '#prefix' do
- it 'returns default prefix when not specified' do
- expect(client.prefix).to eq('gitlab_active_context')
- end
-
- it 'returns configured prefix' do
- client = described_class.new(options.merge(prefix: 'custom'))
- expect(client.prefix).to eq('custom')
- end
- end
end
diff --git a/gems/gitlab-active-context/spec/lib/active_context/databases/opensearch/adapter_spec.rb b/gems/gitlab-active-context/spec/lib/active_context/databases/opensearch/adapter_spec.rb
index 1afba9a992b1e5954085d9c732c8b08c27f0dbbe..859cb28f34913de34ffeb96aade0f01367683e93 100644
--- a/gems/gitlab-active-context/spec/lib/active_context/databases/opensearch/adapter_spec.rb
+++ b/gems/gitlab-active-context/spec/lib/active_context/databases/opensearch/adapter_spec.rb
@@ -11,4 +11,15 @@
adapter.search(query)
end
+
+ describe '#prefix' do
+ it 'returns default prefix when not specified' do
+ expect(adapter.prefix).to eq('gitlab_active_context')
+ end
+
+ it 'returns configured prefix' do
+ adapter = described_class.new(options.merge(prefix: 'custom'))
+ expect(adapter.prefix).to eq('custom')
+ end
+ end
end
diff --git a/gems/gitlab-active-context/spec/lib/active_context/databases/opensearch/client_spec.rb b/gems/gitlab-active-context/spec/lib/active_context/databases/opensearch/client_spec.rb
index 40cba13c9333ef039646d6343cecc6b203223e5d..ee81655a295c832a4021e0ed8f70abc1af3f4027 100644
--- a/gems/gitlab-active-context/spec/lib/active_context/databases/opensearch/client_spec.rb
+++ b/gems/gitlab-active-context/spec/lib/active_context/databases/opensearch/client_spec.rb
@@ -78,15 +78,4 @@
end
end
end
-
- describe '#prefix' do
- it 'returns default prefix when not specified' do
- expect(client.prefix).to eq('gitlab_active_context')
- end
-
- it 'returns configured prefix' do
- client = described_class.new(options.merge(prefix: 'custom'))
- expect(client.prefix).to eq('custom')
- end
- end
end
diff --git a/gems/gitlab-active-context/spec/lib/active_context/databases/postgresql/adapter_spec.rb b/gems/gitlab-active-context/spec/lib/active_context/databases/postgresql/adapter_spec.rb
index 059efe66f485596a9753a878ac18a3e7540d6b22..0c0628cf3b7a32851914326a09aa71dcc3cb17c7 100644
--- a/gems/gitlab-active-context/spec/lib/active_context/databases/postgresql/adapter_spec.rb
+++ b/gems/gitlab-active-context/spec/lib/active_context/databases/postgresql/adapter_spec.rb
@@ -19,4 +19,15 @@
adapter.search(query)
end
+
+ describe '#prefix' do
+ it 'returns default prefix when not specified' do
+ expect(adapter.prefix).to eq('gitlab_active_context')
+ end
+
+ it 'returns configured prefix' do
+ adapter = described_class.new(options.merge(prefix: 'custom'))
+ expect(adapter.prefix).to eq('custom')
+ end
+ end
end
diff --git a/gems/gitlab-active-context/spec/lib/active_context/databases/postgresql/client_spec.rb b/gems/gitlab-active-context/spec/lib/active_context/databases/postgresql/client_spec.rb
index 6e1ee75c79135b3e9dc548cd01f4eaaf4eb06384..738ba011f30b90cb7c0fd3afde3ff5dc2393db9f 100644
--- a/gems/gitlab-active-context/spec/lib/active_context/databases/postgresql/client_spec.rb
+++ b/gems/gitlab-active-context/spec/lib/active_context/databases/postgresql/client_spec.rb
@@ -16,40 +16,131 @@
subject(:client) { described_class.new(options) }
describe '#initialize' do
- it 'creates a connection pool' do
- expect(ConnectionPool).to receive(:new)
- .with(hash_including(size: 2, timeout: 1))
+ let(:connection_pool) { instance_double(ActiveRecord::ConnectionAdapters::ConnectionPool) }
+ let(:connection_model) { class_double(ActiveRecord::Base) }
+
+ before do
+ allow_any_instance_of(described_class).to receive(:create_connection_model)
+ .and_return(connection_model)
+
+ allow(connection_model).to receive(:establish_connection)
+ allow(connection_model).to receive(:connection_pool).and_return(connection_pool)
+ end
+
+ it 'creates a connection pool through ActiveRecord' do
+ expected_config = {
+ 'adapter' => 'postgresql',
+ 'host' => 'localhost',
+ 'port' => 5432,
+ 'database' => 'test_db',
+ 'username' => 'user',
+ 'password' => 'pass',
+ 'connect_timeout' => 5,
+ 'pool' => 2,
+ 'prepared_statements' => false,
+ 'advisory_locks' => false,
+ 'database_tasks' => false
+ }
+
+ expect(connection_model).to receive(:establish_connection)
+ .with(hash_including(expected_config))
client
end
end
+ describe '#with_raw_connection' do
+ let(:raw_connection) { instance_double(PG::Connection) }
+ let(:connection_pool) { instance_double(ActiveRecord::ConnectionAdapters::ConnectionPool) }
+ let(:ar_connection) { instance_double(ActiveRecord::ConnectionAdapters::PostgreSQLAdapter) }
+ let(:connection_model) { class_double(ActiveRecord::Base) }
+ let(:yielded_values) { [] }
+
+ before do
+ allow_any_instance_of(described_class).to receive(:create_connection_model)
+ .and_return(connection_model)
+
+ allow(connection_model).to receive(:establish_connection)
+ allow(connection_model).to receive(:connection_pool).and_return(connection_pool)
+
+ allow(connection_pool).to receive(:with_connection).and_yield(ar_connection)
+ allow(ar_connection).to receive_messages(
+ raw_connection: raw_connection
+ )
+
+ allow(raw_connection).to receive(:server_version).and_return(120000)
+ end
+
+ it 'yields raw PostgreSQL connection' do
+ client.with_raw_connection do |conn|
+ yielded_values << conn
+ end
+
+ expect(yielded_values).to eq([raw_connection])
+ end
+ end
+
+ describe '#with_connection' do
+ let(:raw_connection) { instance_double(PG::Connection) }
+ let(:connection_pool) { instance_double(ActiveRecord::ConnectionAdapters::ConnectionPool) }
+ let(:ar_connection) { instance_double(ActiveRecord::ConnectionAdapters::PostgreSQLAdapter) }
+ let(:connection_model) { class_double(ActiveRecord::Base) }
+ let(:yielded_values) { [] }
+
+ before do
+ allow_any_instance_of(described_class).to receive(:create_connection_model)
+ .and_return(connection_model)
+
+ allow(connection_model).to receive(:establish_connection)
+ allow(connection_model).to receive(:connection_pool).and_return(connection_pool)
+
+ allow(connection_pool).to receive(:with_connection).and_yield(ar_connection)
+ allow(ar_connection).to receive_messages(
+ raw_connection: raw_connection
+ )
+
+ allow(raw_connection).to receive(:server_version).and_return(120000)
+ end
+
+ it 'yields ActiveRecord connection' do
+ client.with_connection do |conn|
+ yielded_values << conn
+ end
+
+ expect(yielded_values).to eq([ar_connection])
+ end
+ end
+
describe '#search' do
- let(:connection) { instance_double(PG::Connection) }
+ let(:raw_connection) { instance_double(PG::Connection) }
let(:query_result) { instance_double(PG::Result) }
+ let(:connection_pool) { instance_double(ActiveRecord::ConnectionAdapters::ConnectionPool) }
+ let(:ar_connection) { instance_double(ActiveRecord::ConnectionAdapters::PostgreSQLAdapter) }
+ let(:connection_model) { class_double(ActiveRecord::Base) }
before do
- allow(PG).to receive(:connect).and_return(connection)
- allow(connection).to receive(:exec).and_return(query_result)
+ allow_any_instance_of(described_class).to receive(:create_connection_model)
+ .and_return(connection_model)
+
+ allow(connection_model).to receive(:establish_connection)
+ allow(connection_model).to receive(:connection_pool).and_return(connection_pool)
+
+ allow(connection_pool).to receive(:with_connection).and_yield(ar_connection)
+ allow(ar_connection).to receive_messages(
+ execute: query_result,
+ raw_connection: raw_connection
+ )
+
+ allow(raw_connection).to receive(:server_version).and_return(120000)
+ allow(ActiveContext::Databases::Postgresql::QueryResult).to receive(:new)
end
it 'executes query and returns QueryResult' do
- expect(connection).to receive(:exec).with('SELECT * FROM pg_stat_activity')
+ expect(ar_connection).to receive(:execute).with('SELECT * FROM pg_stat_activity')
expect(ActiveContext::Databases::Postgresql::QueryResult)
.to receive(:new).with(query_result)
client.search('test query')
end
end
-
- describe '#prefix' do
- it 'returns default prefix when not specified' do
- expect(client.prefix).to eq('gitlab_active_context')
- end
-
- it 'returns configured prefix' do
- client = described_class.new(options.merge(prefix: 'custom'))
- expect(client.prefix).to eq('custom')
- end
- end
end