From d3e207e8ee93348d6b99b8c1f5a4ac538dac2eda Mon Sep 17 00:00:00 2001
From: Max Woolf <mwoolf@gitlab.com>
Date: Fri, 5 May 2023 10:25:49 +0000
Subject: [PATCH] Add RedisHLL counter for creating dashboards
On push, we enqueue a new worker to check
if a new analytics dashboard was created and
increments a project-specific counter.
This will give us a count of the number of projects
with at least one dashboard.
EE: true
Changelog: added
---
config/sidekiq_queues.yml | 2 +
ee/app/services/ee/git/branch_push_service.rb | 8 +++
ee/app/workers/all_queues.yml | 9 ++++
.../product_analytics/post_push_worker.rb | 33 ++++++++++++
...jects_with_analytics_dashboard_monthly.yml | 21 ++++++++
...ojects_with_analytics_dashboard_weekly.yml | 21 ++++++++
.../ee/git/branch_push_service_spec.rb | 33 ++++++++++++
.../post_push_worker_spec.rb | 54 +++++++++++++++++++
.../known_events/product_analytics.yml | 2 +
9 files changed, 183 insertions(+)
create mode 100644 ee/app/workers/product_analytics/post_push_worker.rb
create mode 100644 ee/config/metrics/counts_28d/20230504085937_projects_with_analytics_dashboard_monthly.yml
create mode 100644 ee/config/metrics/counts_7d/20230504085937_projects_with_analytics_dashboard_weekly.yml
create mode 100644 ee/spec/workers/product_analytics/post_push_worker_spec.rb
create mode 100644 lib/gitlab/usage_data_counters/known_events/product_analytics.yml
diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml
index f0917c9265b01..5a7e0d41e49fb 100644
--- a/config/sidekiq_queues.yml
+++ b/config/sidekiq_queues.yml
@@ -421,6 +421,8 @@
- 1
- - product_analytics_initialize_snowplow_product_analytics
- 1
+- - product_analytics_post_push
+ - 1
- - project_cache
- 1
- - project_destroy
diff --git a/ee/app/services/ee/git/branch_push_service.rb b/ee/app/services/ee/git/branch_push_service.rb
index 99e9316d869e2..67f2a98d0e73c 100644
--- a/ee/app/services/ee/git/branch_push_service.rb
+++ b/ee/app/services/ee/git/branch_push_service.rb
@@ -10,12 +10,20 @@ def execute
enqueue_elasticsearch_indexing
enqueue_zoekt_indexing
enqueue_update_external_pull_requests
+ enqueue_product_analytics_event_metrics
super
end
private
+ def enqueue_product_analytics_event_metrics
+ return unless project.product_analytics_enabled?
+ return unless default_branch?
+
+ ::ProductAnalytics::PostPushWorker.perform_async(project.id, newrev)
+ end
+
def enqueue_elasticsearch_indexing
return unless should_index_commits?
diff --git a/ee/app/workers/all_queues.yml b/ee/app/workers/all_queues.yml
index 647cf4907ce53..3d3ce3737071a 100644
--- a/ee/app/workers/all_queues.yml
+++ b/ee/app/workers/all_queues.yml
@@ -1524,6 +1524,15 @@
:weight: 1
:idempotent: true
:tags: []
+- :name: product_analytics_post_push
+ :worker_name: ProductAnalytics::PostPushWorker
+ :feature_category: :product_analytics
+ :has_external_dependencies: false
+ :urgency: :low
+ :resource_boundary: :unknown
+ :weight: 1
+ :idempotent: true
+ :tags: []
- :name: project_import_schedule
:worker_name: ProjectImportScheduleWorker
:feature_category: :source_code_management
diff --git a/ee/app/workers/product_analytics/post_push_worker.rb b/ee/app/workers/product_analytics/post_push_worker.rb
new file mode 100644
index 0000000000000..0b8c4bcdac81f
--- /dev/null
+++ b/ee/app/workers/product_analytics/post_push_worker.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module ProductAnalytics
+ class PostPushWorker
+ include ApplicationWorker
+
+ data_consistency :sticky
+ feature_category :product_analytics
+ idempotent!
+
+ def perform(project_id, newrev)
+ @project = Project.find_by_id(project_id)
+ @commit = @project.repository.commit(newrev)
+
+ track_event if commit_has_new_dashboard?
+ end
+
+ private
+
+ def commit_has_new_dashboard?
+ @commit.deltas.any? do |delta|
+ delta.new_path.start_with?(".gitlab/analytics/dashboards/") && delta.new_file
+ end
+ end
+
+ def track_event
+ Gitlab::UsageDataCounters::HLLRedisCounter.track_usage_event(
+ :project_created_analytics_dashboard,
+ @project.id
+ )
+ end
+ end
+end
diff --git a/ee/config/metrics/counts_28d/20230504085937_projects_with_analytics_dashboard_monthly.yml b/ee/config/metrics/counts_28d/20230504085937_projects_with_analytics_dashboard_monthly.yml
new file mode 100644
index 0000000000000..02571ca4918eb
--- /dev/null
+++ b/ee/config/metrics/counts_28d/20230504085937_projects_with_analytics_dashboard_monthly.yml
@@ -0,0 +1,21 @@
+---
+key_path: counts.projects_with_analytics_dashboard_monthly
+description: Count of projects with an analytics dashboard
+product_section: dev
+product_stage: analyze
+product_group: product_analytics
+value_type: number
+status: active
+milestone: "16.0"
+introduced_by_url:
+time_frame: 28d
+data_source: redis_hll
+data_category: optional
+instrumentation_class: RedisHLLMetric
+distribution:
+- ee
+tier:
+- ultimate
+options:
+ events:
+ - project_created_analytics_dashboard
diff --git a/ee/config/metrics/counts_7d/20230504085937_projects_with_analytics_dashboard_weekly.yml b/ee/config/metrics/counts_7d/20230504085937_projects_with_analytics_dashboard_weekly.yml
new file mode 100644
index 0000000000000..482e6a48a78a0
--- /dev/null
+++ b/ee/config/metrics/counts_7d/20230504085937_projects_with_analytics_dashboard_weekly.yml
@@ -0,0 +1,21 @@
+---
+key_path: counts.projects_with_analytics_dashboard_weekly
+description: Count of projects with an analytics dashboard
+product_section: dev
+product_stage: analyze
+product_group: product_analytics
+value_type: number
+status: active
+milestone: "16.0"
+introduced_by_url:
+time_frame: 7d
+data_source: redis_hll
+data_category: optional
+instrumentation_class: RedisHLLMetric
+distribution:
+- ee
+tier:
+- ultimate
+options:
+ events:
+ - project_created_analytics_dashboard
diff --git a/ee/spec/services/ee/git/branch_push_service_spec.rb b/ee/spec/services/ee/git/branch_push_service_spec.rb
index 33d2947ee01cb..822b4ff19424b 100644
--- a/ee/spec/services/ee/git/branch_push_service_spec.rb
+++ b/ee/spec/services/ee/git/branch_push_service_spec.rb
@@ -184,5 +184,38 @@
end
end
end
+
+ context 'Product Analytics' do
+ using RSpec::Parameterized::TableSyntax
+
+ where(:flag_enabled, :default_branch, :licence_available, :called) do
+ true | 'master' | true | true
+ true | 'master' | false | false
+ true | 'other' | true | false
+ true | 'other' | false | false
+ false | 'master' | true | false
+ false | 'master' | false | false
+ false | 'other' | true | false
+ false | 'other' | false | false
+ end
+
+ before do
+ stub_feature_flags(product_analytics_dashboards: flag_enabled)
+ stub_licensed_features(product_analytics: licence_available)
+ allow(project).to receive(:default_branch).and_return(default_branch)
+ end
+
+ with_them do
+ it 'enqueues the worker if appropriate' do
+ if called
+ expect(::ProductAnalytics::PostPushWorker).to receive(:perform_async).once
+ else
+ expect(::ProductAnalytics::PostPushWorker).not_to receive(:perform_async)
+ end
+
+ subject.execute
+ end
+ end
+ end
end
end
diff --git a/ee/spec/workers/product_analytics/post_push_worker_spec.rb b/ee/spec/workers/product_analytics/post_push_worker_spec.rb
new file mode 100644
index 0000000000000..37a9918797f42
--- /dev/null
+++ b/ee/spec/workers/product_analytics/post_push_worker_spec.rb
@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe ProductAnalytics::PostPushWorker, feature_category: :product_analytics do
+ include RepoHelpers
+
+ let_it_be(:project) { create(:project, :repository) }
+
+ let(:commit) { project.repository.commit }
+
+ subject { described_class.new.perform(project.id, commit.sha) }
+
+ shared_examples 'tracks a usage event' do
+ it 'tracks a usage event' do
+ expect(Gitlab::UsageDataCounters::HLLRedisCounter)
+ .to receive(:track_usage_event).with(:project_created_analytics_dashboard, project.id)
+
+ subject
+ end
+ end
+
+ shared_examples 'does not track a usage event' do
+ it 'does not track a usage event' do
+ expect(Gitlab::Utils::UsageData).not_to receive(:track_usage_event)
+
+ subject
+ end
+ end
+
+ context 'when the commit includes a new dashboard' do
+ before do
+ create_new_dashboard
+ end
+
+ it_behaves_like 'tracks a usage event'
+ end
+
+ context 'when the commit includes a new file that is not a dashboard' do
+ it_behaves_like 'does not track a usage event'
+ end
+
+ private
+
+ def create_new_dashboard
+ project.repository.create_file(
+ project.creator,
+ '.gitlab/analytics/dashboards/dashboard_hello/dashboard_hello.yml',
+ 'content',
+ message: 'Add dashboard',
+ branch_name: 'master'
+ )
+ end
+end
diff --git a/lib/gitlab/usage_data_counters/known_events/product_analytics.yml b/lib/gitlab/usage_data_counters/known_events/product_analytics.yml
new file mode 100644
index 0000000000000..b61e2f4e5a2b5
--- /dev/null
+++ b/lib/gitlab/usage_data_counters/known_events/product_analytics.yml
@@ -0,0 +1,2 @@
+- name: project_created_analytics_dashboard
+ aggregation: weekly
--
GitLab