Merge branch 'change-catalog-resource-last-30-day-updated-at-default' into 'master'

Change catalog resource last_30_day_usage_count_updated_at default

See merge request https://gitlab.com/gitlab-org/gitlab/-/merge_requests/156575



Merged-by: Leaminn Ma <lma@gitlab.com>
Approved-by: John Mason <9717668-johnmason@users.noreply.gitlab.com>
Approved-by: Adam Hegyi <ahegyi@gitlab.com>
Approved-by: Laura Montemayor <lmontemayor@gitlab.com>

app/services/ci/catalog/resources/aggregate_last30_day_usage_service.rb

@@ -38,6 +38,8 @@ def execute
 
         private
 
+        # NOTE: New catalog resources added today are considered already processed
+        # because their `last_30_day_usage_count_updated_at` is defaulted to NOW().
         def done_processing?
           min_updated_at = TARGET_MODEL.minimum(:last_30_day_usage_count_updated_at)
           return true unless min_updated_at

db/post_migrate/20240617210449_change_catalog_resources_last30_day_usage_count_updated_at_default.rb

+# frozen_string_literal: true
+
+class ChangeCatalogResourcesLast30DayUsageCountUpdatedAtDefault < Gitlab::Database::Migration[2.2]
+  milestone '17.1'
+
+  def up
+    change_column_default :catalog_resources, :last_30_day_usage_count_updated_at, -> { 'NOW()' }
+  end
+
+  def down
+    change_column_default :catalog_resources, :last_30_day_usage_count_updated_at, '1970-01-01'
+  end
+end

db/schema_migrations/20240617210449

+2f66063d4b1ec670ec8f1e1c2d471ad6d85fc45150e932454fe85390565d3e34
\ No newline at end of file

db/structure.sql

@@ -6984,7 +6984,7 @@ CREATE TABLE catalog_resources (
     search_vector tsvector GENERATED ALWAYS AS ((setweight(to_tsvector('english'::regconfig, (COALESCE(name, ''::character varying))::text), 'A'::"char") || setweight(to_tsvector('english'::regconfig, COALESCE(description, ''::text)), 'B'::"char"))) STORED,
     verification_level smallint DEFAULT 0,
     last_30_day_usage_count integer DEFAULT 0 NOT NULL,
-    last_30_day_usage_count_updated_at timestamp with time zone DEFAULT '1970-01-01 00:00:00+00'::timestamp with time zone NOT NULL
+    last_30_day_usage_count_updated_at timestamp with time zone DEFAULT now() NOT NULL
 );
 CREATE SEQUENCE catalog_resources_id_seq

lib/gitlab/ci/components/usages/aggregator.rb

@@ -65,7 +65,10 @@ class Aggregator
           TARGET_BATCH_SIZE = 1000
           DISTINCT_USAGE_BATCH_SIZE = 100
           MAX_RUNTIME = 4.minutes # Should be >= job scheduling frequency so there is no gap between job runs
-          WORKER_DEDUP_TTL = MAX_RUNTIME + 1.minute # Includes extra time to execute `&usage_counts_block`
+
+          # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/155001#note_1941066672
+          # Includes extra time (1.minute) to execute `&usage_counts_block`
+          WORKER_DEDUP_TTL = MAX_RUNTIME + 1.minute
           LEASE_TIMEOUT = 10.minutes
 
           def initialize(target_model:, group_by_column:, usage_start_date:, usage_end_date:, lease_key:)

spec/services/ci/catalog/resources/aggregate_last30_day_usage_service_spec.rb

@@ -125,6 +125,19 @@
           expect(response.message).to eq("Processing complete for #{Date.today}")
           expect(response.payload).to eq({})
         end
+
+        context 'when a new catalog resource is added today' do
+          it 'does not aggregate usage data' do
+            create(:ci_catalog_resource)
+            expect(Gitlab::Ci::Components::Usages::Aggregator).not_to receive(:new)
+
+            response = service.execute
+
+            expect(response).to be_success
+            expect(response.message).to eq("Processing complete for #{Date.today}")
+            expect(response.payload).to eq({})
+          end
+        end
       end
     end
 

spec/workers/ci/catalog/resources/aggregate_last30_day_usage_worker_spec.rb

@@ -39,6 +39,8 @@
             component: component, used_date: usage_start_date, used_by_project_id: k)
         end
       end
+
+      Ci::Catalog::Resource.update_all(last_30_day_usage_count_updated_at: usage_end_date.to_time)
     end
 
     it 'aggregates and updates usage counts for all catalog resources' do