diff --git a/app/services/ci/catalog/resources/aggregate_last30_day_usage_service.rb b/app/services/ci/catalog/resources/aggregate_last30_day_usage_service.rb index b6cf7ed6990c76490fdd83b3b3bc4c1e7e91bc30..076140c1a85e9b5b7fb1e32cb9f6c041c36169f6 100644 --- a/app/services/ci/catalog/resources/aggregate_last30_day_usage_service.rb +++ b/app/services/ci/catalog/resources/aggregate_last30_day_usage_service.rb @@ -38,6 +38,8 @@ def execute private + # NOTE: New catalog resources added today are considered already processed + # because their `last_30_day_usage_count_updated_at` is defaulted to NOW(). def done_processing? min_updated_at = TARGET_MODEL.minimum(:last_30_day_usage_count_updated_at) return true unless min_updated_at diff --git a/db/post_migrate/20240617210449_change_catalog_resources_last30_day_usage_count_updated_at_default.rb b/db/post_migrate/20240617210449_change_catalog_resources_last30_day_usage_count_updated_at_default.rb new file mode 100644 index 0000000000000000000000000000000000000000..7c46a87088ebd24764375d7a0294d9439b864d3c --- /dev/null +++ b/db/post_migrate/20240617210449_change_catalog_resources_last30_day_usage_count_updated_at_default.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class ChangeCatalogResourcesLast30DayUsageCountUpdatedAtDefault < Gitlab::Database::Migration[2.2] + milestone '17.1' + + def up + change_column_default :catalog_resources, :last_30_day_usage_count_updated_at, -> { 'NOW()' } + end + + def down + change_column_default :catalog_resources, :last_30_day_usage_count_updated_at, '1970-01-01' + end +end diff --git a/db/schema_migrations/20240617210449 b/db/schema_migrations/20240617210449 new file mode 100644 index 0000000000000000000000000000000000000000..7dac2a07747c6bcbf962eb45b5b18ea221346d4b --- /dev/null +++ b/db/schema_migrations/20240617210449 @@ -0,0 +1 @@ +2f66063d4b1ec670ec8f1e1c2d471ad6d85fc45150e932454fe85390565d3e34 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 4552ab02f5f18c2f1ddef319ed5d3828ca60bb17..2b6919a84bc185a5003884eb21cb19ba86e1ba90 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -6984,7 +6984,7 @@ CREATE TABLE catalog_resources ( search_vector tsvector GENERATED ALWAYS AS ((setweight(to_tsvector('english'::regconfig, (COALESCE(name, ''::character varying))::text), 'A'::"char") || setweight(to_tsvector('english'::regconfig, COALESCE(description, ''::text)), 'B'::"char"))) STORED, verification_level smallint DEFAULT 0, last_30_day_usage_count integer DEFAULT 0 NOT NULL, - last_30_day_usage_count_updated_at timestamp with time zone DEFAULT '1970-01-01 00:00:00+00'::timestamp with time zone NOT NULL + last_30_day_usage_count_updated_at timestamp with time zone DEFAULT now() NOT NULL ); CREATE SEQUENCE catalog_resources_id_seq diff --git a/lib/gitlab/ci/components/usages/aggregator.rb b/lib/gitlab/ci/components/usages/aggregator.rb index ddb6121831a1ccc61bf5aebbd808142fb5e89cf2..7650e6b4fbbdabc93ca5024378e5c417c4735a1a 100644 --- a/lib/gitlab/ci/components/usages/aggregator.rb +++ b/lib/gitlab/ci/components/usages/aggregator.rb @@ -65,7 +65,10 @@ class Aggregator TARGET_BATCH_SIZE = 1000 DISTINCT_USAGE_BATCH_SIZE = 100 MAX_RUNTIME = 4.minutes # Should be >= job scheduling frequency so there is no gap between job runs - WORKER_DEDUP_TTL = MAX_RUNTIME + 1.minute # Includes extra time to execute `&usage_counts_block` + + # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/155001#note_1941066672 + # Includes extra time (1.minute) to execute `&usage_counts_block` + WORKER_DEDUP_TTL = MAX_RUNTIME + 1.minute LEASE_TIMEOUT = 10.minutes def initialize(target_model:, group_by_column:, usage_start_date:, usage_end_date:, lease_key:) diff --git a/spec/services/ci/catalog/resources/aggregate_last30_day_usage_service_spec.rb b/spec/services/ci/catalog/resources/aggregate_last30_day_usage_service_spec.rb index 47d2a3dff3308d24bc2749a17fe3f8d22456b9be..5ee71930d2a6cabfe65f7a4477a8f50d0cc336fc 100644 --- a/spec/services/ci/catalog/resources/aggregate_last30_day_usage_service_spec.rb +++ b/spec/services/ci/catalog/resources/aggregate_last30_day_usage_service_spec.rb @@ -125,6 +125,19 @@ expect(response.message).to eq("Processing complete for #{Date.today}") expect(response.payload).to eq({}) end + + context 'when a new catalog resource is added today' do + it 'does not aggregate usage data' do + create(:ci_catalog_resource) + expect(Gitlab::Ci::Components::Usages::Aggregator).not_to receive(:new) + + response = service.execute + + expect(response).to be_success + expect(response.message).to eq("Processing complete for #{Date.today}") + expect(response.payload).to eq({}) + end + end end end diff --git a/spec/workers/ci/catalog/resources/aggregate_last30_day_usage_worker_spec.rb b/spec/workers/ci/catalog/resources/aggregate_last30_day_usage_worker_spec.rb index 973969f58a268194629f44360b596eaf2ab9f1be..4bc76fbadae8b8718ff3250a8ef9b4d2385c46d9 100644 --- a/spec/workers/ci/catalog/resources/aggregate_last30_day_usage_worker_spec.rb +++ b/spec/workers/ci/catalog/resources/aggregate_last30_day_usage_worker_spec.rb @@ -39,6 +39,8 @@ component: component, used_date: usage_start_date, used_by_project_id: k) end end + + Ci::Catalog::Resource.update_all(last_30_day_usage_count_updated_at: usage_end_date.to_time) end it 'aggregates and updates usage counts for all catalog resources' do