Merge branch 'bg-migration-environment-tier' into 'master'

Background migration to backfill environment tiers

Closes gitlab-org/cluster-integration/gitlab-agent#332

See merge request https://gitlab.com/gitlab-org/gitlab/-/merge_requests/105998



Merged-by: Alex Ives <aives@gitlab.com>
Approved-by: Alishan Ladhani <aladhani@gitlab.com>
Approved-by: Allen Cook <acook@gitlab.com>
Approved-by: Vitali Tatarintev <vtatarintev@gitlab.com>
Approved-by: Alex Ives <aives@gitlab.com>
Reviewed-by: Alex Ives <aives@gitlab.com>
Co-authored-by: Shinya Maeda <shinya@gitlab.com>

app/models/environment.rb

@@ -48,9 +48,9 @@ class Environment < ApplicationRecord
   has_one :latest_opened_most_severe_alert, -> { order_severity_with_open_prometheus_alert }, class_name: 'AlertManagement::Alert', inverse_of: :environment
 
   before_validation :generate_slug, if: ->(env) { env.slug.blank? }
+  before_validation :ensure_environment_tier
 
   before_save :set_environment_type
-  before_save :ensure_environment_tier
   after_save :clear_reactive_cache!
 
   validates :name,
@@ -71,6 +71,10 @@ class Environment < ApplicationRecord
             length: { maximum: 255 },
             allow_nil: true
 
+  # Currently, the tier presence is validaed for newly created environments.
+  # After the `BackfillEnvironmentTiers` background migration has been completed, we should remove `on: :create`.
+  # See https://gitlab.com/gitlab-org/gitlab/-/issues/385253.
+  validates :tier, presence: true, on: :create
   validate :safe_external_url
   validate :merge_request_not_changed
 

db/post_migrate/20221205151917_schedule_backfill_environment_tier.rb

+# frozen_string_literal: true
+
+class ScheduleBackfillEnvironmentTier < Gitlab::Database::Migration[2.0]
+  MIGRATION = 'BackfillEnvironmentTiers'
+  DELAY_INTERVAL = 2.minutes
+
+  restrict_gitlab_migration gitlab_schema: :gitlab_main
+
+  def up
+    queue_batched_background_migration(
+      MIGRATION,
+      :environments,
+      :id,
+      job_interval: DELAY_INTERVAL
+    )
+  end
+
+  def down
+    delete_batched_background_migration(MIGRATION, :environments, :id, [])
+  end
+end

db/schema_migrations/20221205151917

+64c4d48759a28b2dbac035d290724ee82b8af4af2bef75e1d7e07eefe10e6d17
\ No newline at end of file

lib/gitlab/background_migration/backfill_environment_tiers.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # This class backfills the `environments.tier` column by using `guess_tier` logic.
+    # Environments created after 13.10 already have a value, however, environments created before 13.10 don't.
+    # See https://gitlab.com/gitlab-org/gitlab/-/issues/300741 for more information.
+    class BackfillEnvironmentTiers < BatchedMigrationJob
+      operation_name :backfill_environment_tiers
+
+      # Equivalent to `Environment#guess_tier` pattern matching.
+      PRODUCTION_TIER = 0
+      STAGING_TIER = 1
+      TESTING_TIER = 2
+      DEVELOPMENT_TIER = 3
+      OTHER_TIER = 4
+
+      TIER_REGEXP_PAIR = [
+        { tier: DEVELOPMENT_TIER, regexp: '(dev|review|trunk)' },
+        { tier: TESTING_TIER, regexp: '(test|tst|int|ac(ce|)pt|qa|qc|control|quality)' },
+        { tier: STAGING_TIER, regexp: '(st(a|)g|mod(e|)l|pre|demo|non)' },
+        { tier: PRODUCTION_TIER, regexp: '(pr(o|)d|live)' }
+      ].freeze
+
+      def perform
+        TIER_REGEXP_PAIR.each do |pair|
+          each_sub_batch(
+            batching_scope: ->(relation) { relation.where(tier: nil).where("name ~* '#{pair[:regexp]}'") } # rubocop:disable GitlabSecurity/SqlInjection
+          ) do |sub_batch|
+            sub_batch.update_all(tier: pair[:tier])
+          end
+        end
+
+        each_sub_batch(batching_scope: ->(relation) { relation.where(tier: nil) }) do |sub_batch|
+          sub_batch.update_all(tier: OTHER_TIER)
+        end
+      end
+    end
+  end
+end

spec/lib/gitlab/background_migration/backfill_environment_tiers_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::BackfillEnvironmentTiers,
+  :migration, schema: 20221205151917, feature_category: :continuous_delivery do
+  let!(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') }
+  let!(:project) { table(:projects).create!(namespace_id: namespace.id, project_namespace_id: namespace.id) }
+
+  let(:migration) do
+    described_class.new(start_id: 1, end_id: 1000,
+                        batch_table: :environments, batch_column: :id,
+                        sub_batch_size: 10, pause_ms: 0,
+                        connection: ApplicationRecord.connection)
+  end
+
+  describe '#perform' do
+    let!(:production) { table(:environments).create!(name: 'production', slug: 'production', project_id: project.id) }
+    let!(:staging) { table(:environments).create!(name: 'staging', slug: 'staging', project_id: project.id) }
+    let!(:testing) { table(:environments).create!(name: 'testing', slug: 'testing', project_id: project.id) }
+
+    let!(:development) do
+      table(:environments).create!(name: 'development', slug: 'development', project_id: project.id)
+    end
+
+    let!(:other) { table(:environments).create!(name: 'other', slug: 'other', project_id: project.id) }
+
+    it 'backfill tiers for all environments in range' do
+      expect(production.tier).to be_nil
+      expect(staging.tier).to be_nil
+      expect(testing.tier).to be_nil
+      expect(development.tier).to be_nil
+      expect(other.tier).to be_nil
+
+      migration.perform
+
+      expect(production.reload.tier).to eq(described_class::PRODUCTION_TIER)
+      expect(staging.reload.tier).to eq(described_class::STAGING_TIER)
+      expect(testing.reload.tier).to eq(described_class::TESTING_TIER)
+      expect(development.reload.tier).to eq(described_class::DEVELOPMENT_TIER)
+      expect(other.reload.tier).to eq(described_class::OTHER_TIER)
+    end
+  end
+
+  # Equivalent to spec/models/environment_spec.rb#guess_tier
+  describe 'same behavior with guess tier' do
+    using RSpec::Parameterized::TableSyntax
+
+    let(:environment) { table(:environments).create!(name: name, slug: name, project_id: project.id) }
+
+    where(:name, :tier) do
+      'review/feature'     | described_class::DEVELOPMENT_TIER
+      'review/product'     | described_class::DEVELOPMENT_TIER
+      'DEV'                | described_class::DEVELOPMENT_TIER
+      'development'        | described_class::DEVELOPMENT_TIER
+      'trunk'              | described_class::DEVELOPMENT_TIER
+      'dev'                | described_class::DEVELOPMENT_TIER
+      'review/app'         | described_class::DEVELOPMENT_TIER
+      'PRODUCTION'         | described_class::PRODUCTION_TIER
+      'prod'               | described_class::PRODUCTION_TIER
+      'prod-east-2'        | described_class::PRODUCTION_TIER
+      'us-prod-east'       | described_class::PRODUCTION_TIER
+      'fe-production'      | described_class::PRODUCTION_TIER
+      'test'               | described_class::TESTING_TIER
+      'TEST'               | described_class::TESTING_TIER
+      'testing'            | described_class::TESTING_TIER
+      'testing-prd'        | described_class::TESTING_TIER
+      'acceptance-testing' | described_class::TESTING_TIER
+      'production-test'    | described_class::TESTING_TIER
+      'test-production'    | described_class::TESTING_TIER
+      'QC'                 | described_class::TESTING_TIER
+      'qa-env-2'           | described_class::TESTING_TIER
+      'gstg'               | described_class::STAGING_TIER
+      'staging'            | described_class::STAGING_TIER
+      'stage'              | described_class::STAGING_TIER
+      'Model'              | described_class::STAGING_TIER
+      'MODL'               | described_class::STAGING_TIER
+      'Pre-production'     | described_class::STAGING_TIER
+      'pre'                | described_class::STAGING_TIER
+      'Demo'               | described_class::STAGING_TIER
+      'staging'            | described_class::STAGING_TIER
+      'pre-prod'           | described_class::STAGING_TIER
+      'blue-kit-stage'     | described_class::STAGING_TIER
+      'nonprod'            | described_class::STAGING_TIER
+      'nonlive'            | described_class::STAGING_TIER
+      'non-prod'           | described_class::STAGING_TIER
+      'non-live'           | described_class::STAGING_TIER
+      'gprd'               | described_class::PRODUCTION_TIER
+      'gprd-cny'           | described_class::PRODUCTION_TIER
+      'production'         | described_class::PRODUCTION_TIER
+      'Production'         | described_class::PRODUCTION_TIER
+      'PRODUCTION'         | described_class::PRODUCTION_TIER
+      'Production/eu'      | described_class::PRODUCTION_TIER
+      'production/eu'      | described_class::PRODUCTION_TIER
+      'PRODUCTION/EU'      | described_class::PRODUCTION_TIER
+      'productioneu'       | described_class::PRODUCTION_TIER
+      'store-produce'      | described_class::PRODUCTION_TIER
+      'unproductive'       | described_class::PRODUCTION_TIER
+      'production/www.gitlab.com' | described_class::PRODUCTION_TIER
+      'prod'               | described_class::PRODUCTION_TIER
+      'PROD'               | described_class::PRODUCTION_TIER
+      'Live'               | described_class::PRODUCTION_TIER
+      'canary'             | described_class::OTHER_TIER
+      'other'              | described_class::OTHER_TIER
+      'EXP'                | described_class::OTHER_TIER
+      'something-else'     | described_class::OTHER_TIER
+    end
+
+    with_them do
+      it 'backfill tiers for all environments in range' do
+        expect(environment.tier).to be_nil
+
+        migration.perform
+
+        expect(environment.reload.tier).to eq(tier)
+      end
+    end
+  end
+end

spec/migrations/20221205151917_schedule_backfill_environment_tier_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!
+
+RSpec.describe ScheduleBackfillEnvironmentTier, category: :continuous_delivery do
+  let!(:batched_migration) { described_class::MIGRATION }
+
+  it 'schedules a new batched migration' do
+    reversible_migration do |migration|
+      migration.before -> {
+        expect(batched_migration).not_to have_scheduled_batched_migration
+      }
+
+      migration.after -> {
+        expect(batched_migration).to have_scheduled_batched_migration(
+          table_name: :environments,
+          column_name: :id,
+          interval: described_class::DELAY_INTERVAL
+        )
+      }
+    end
+  end
+end

spec/services/deployments/update_environment_service_spec.rb

@@ -115,7 +115,7 @@
         let(:external_url) { 'javascript:alert("hello")' }
 
         it 'fails to update the tier due to validation error' do
-          expect { subject.execute }.not_to change { environment.tier }
+          expect { subject.execute }.not_to change { environment.reload.tier }
         end
 
         it 'tracks an exception' do