diff --git a/config/feature_flags/wip/redis_hll_property_name_tracking.yml b/config/feature_flags/wip/redis_hll_property_name_tracking.yml new file mode 100644 index 0000000000000000000000000000000000000000..853eb5ce96c4d3a34a094233330219a203cb2f89 --- /dev/null +++ b/config/feature_flags/wip/redis_hll_property_name_tracking.yml @@ -0,0 +1,8 @@ +--- +name: redis_hll_property_name_tracking +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/137890 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/432866 +milestone: '16.9' +type: wip +group: group::analytics instrumentation +default_enabled: false diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index 137b6f90545d6a07041dc6815427d53fd1ed1a88..97039a58e92c9c31c08cbb24a099718f516d33eb 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -29,18 +29,20 @@ class << self # # event_name - The event name. # values - One or multiple values counted. + # property_name - Name of the values counted. # time - Time of the action, set to Time.current. - def track_event(event_name, values:, time: Time.current) - track(values, event_name, time: time) + def track_event(event_name, values:, property_name: nil, time: Time.current) + track(values, event_name, property_name: property_name, time: time) end # Count unique events for a given time range. # # event_names - The list of the events to count. + # property_names - The list of the values for which the events are to be counted. # start_date - The start date of the time range. # end_date - The end date of the time range. - def unique_events(event_names:, start_date:, end_date:) - count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date) + def unique_events(event_names:, start_date:, end_date:, property_name: nil) + count_unique_events(event_names: event_names, property_name: property_name, start_date: start_date, end_date: end_date) end def known_event?(event_name) @@ -52,19 +54,19 @@ def known_events end def calculate_events_union(event_names:, start_date:, end_date:) - count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date) + count_unique_events(event_names: event_names, property_name: nil, start_date: start_date, end_date: end_date) end private - def track(values, event_name, time: Time.zone.now) + def track(values, event_name, property_name:, time: Time.zone.now) event = event_for(event_name) Gitlab::ErrorTracking.track_and_raise_for_dev_exception(UnknownEvent.new("Unknown event #{event_name}")) unless event.present? return if event.blank? return unless Feature.enabled?(:redis_hll_tracking, type: :ops) - Gitlab::Redis::HLL.add(key: redis_key(event, time), value: values, expiry: KEY_EXPIRY_LENGTH) + Gitlab::Redis::HLL.add(key: redis_key(event_with_property_name(event, property_name), time), value: values, expiry: KEY_EXPIRY_LENGTH) rescue StandardError => e # Ignore any exceptions unless is dev or test env @@ -72,8 +74,8 @@ def track(values, event_name, time: Time.zone.now) Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) end - def count_unique_events(event_names:, start_date:, end_date:) - events = events_for(Array(event_names).map(&:to_s)) + def count_unique_events(event_names:, start_date:, end_date:, property_name:) + events = events_with_property_names(event_names, property_name) keys = keys_for_aggregation(events: events, start_date: start_date, end_date: end_date) @@ -82,6 +84,19 @@ def count_unique_events(event_names:, start_date:, end_date:) redis_usage_data { Gitlab::Redis::HLL.count(keys: keys) } end + def events_with_property_names(event_names, property_name) + event_names = Array(event_names).map(&:to_s) + known_events.filter_map do |event| + next unless event_names.include?(event[:name]) + + property_name ? event_with_property_name(event, property_name) : event + end + end + + def event_with_property_name(event, property_name) + event.merge(property_name: property_name) + end + def load_events events = Gitlab::Usage::MetricDefinition.all.map do |d| next unless d.available? @@ -102,21 +117,30 @@ def event_for(event_name) known_events.find { |event| event[:name] == event_name.to_s } end - def events_for(event_names) - known_events.select { |event| event_names.include?(event[:name]) } - end - def redis_key(event, time) - key = redis_key_base(event[:name]) + key = redis_key_base(event) year_week = time.strftime('%G-%V') "{#{REDIS_SLOT}}_#{key}-#{year_week}" end - def redis_key_base(event_name) + def redis_key_base(event) + event_name = event[:name] + raise UnknownEvent, "Unknown event #{event_name}" unless known_events_names.include?(event_name.to_s) - key_overrides.fetch(event_name, event_name) + property_name = event[:property_name] + key = event_name + if Feature.enabled?(:redis_hll_property_name_tracking, type: :wip) && property_name + key = "#{key}-#{formatted_property_name(property_name)}" + end + + key_overrides.fetch(key, key) + end + + def formatted_property_name(property_name) + # simplify to format from EventDefinitions.unique_properties + property_name.to_s.split('.').first.to_sym end def key_overrides diff --git a/lib/gitlab/usage_data_counters/hll_redis_key_overrides.yml b/lib/gitlab/usage_data_counters/hll_redis_key_overrides.yml index 0967ef424bce6791893e9a57bb952f80fd536e93..9adbb02ff2c0df51e8560d9071e10e0a8465c5b1 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_key_overrides.yml +++ b/lib/gitlab/usage_data_counters/hll_redis_key_overrides.yml @@ -1 +1,127 @@ -{} +# This file lists all of the internal events that need to be saved with their legacy HLL Redis keys +# +# This file has been generated using the script included in +# the description of https://gitlab.com/gitlab-org/gitlab/-/merge_requests/137890 +# +# It is only safe to regenerate it using the same script if the +# :redis_hll_property_name_tracking feature flag is disabled on prod environment. +--- +agent_users_using_ci_tunnel-user: agent_users_using_ci_tunnel +ci_template_included-project: ci_template_included +exclude_anonymised_users-user: exclude_anonymised_users +g_compliance_dashboard-user: g_compliance_dashboard +g_edit_by_sfe-user: g_edit_by_sfe +g_edit_by_snippet_ide-user: g_edit_by_snippet_ide +g_edit_by_web_ide-user: g_edit_by_web_ide +g_project_management_epic_blocked_added-user: g_project_management_epic_blocked_added +g_project_management_epic_blocked_removed-user: g_project_management_epic_blocked_removed +g_project_management_epic_blocking_added-user: g_project_management_epic_blocking_added +g_project_management_epic_blocking_removed-user: g_project_management_epic_blocking_removed +g_project_management_epic_closed-user: g_project_management_epic_closed +g_project_management_epic_created-user: g_project_management_epic_created +g_project_management_epic_cross_referenced-user: g_project_management_epic_cross_referenced +g_project_management_epic_destroyed-user: g_project_management_epic_destroyed +g_project_management_epic_issue_added-user: g_project_management_epic_issue_added +g_project_management_epic_issue_moved_from_project-user: g_project_management_epic_issue_moved_from_project +g_project_management_epic_issue_removed-user: g_project_management_epic_issue_removed +g_project_management_epic_related_added-user: g_project_management_epic_related_added +g_project_management_epic_related_removed-user: g_project_management_epic_related_removed +g_project_management_epic_reopened-user: g_project_management_epic_reopened +g_project_management_epic_users_changing_labels-user: g_project_management_epic_users_changing_labels +g_project_management_issue_added_to_epic-user: g_project_management_issue_added_to_epic +g_project_management_issue_assignee_changed-user: g_project_management_issue_assignee_changed +g_project_management_issue_changed_epic-user: g_project_management_issue_changed_epic +g_project_management_issue_cloned-user: g_project_management_issue_cloned +g_project_management_issue_closed-user: g_project_management_issue_closed +g_project_management_issue_comment_added-user: g_project_management_issue_comment_added +g_project_management_issue_comment_edited-user: g_project_management_issue_comment_edited +g_project_management_issue_comment_removed-user: g_project_management_issue_comment_removed +g_project_management_issue_created-user: g_project_management_issue_created +g_project_management_issue_cross_referenced-user: g_project_management_issue_cross_referenced +g_project_management_issue_description_changed-user: g_project_management_issue_description_changed +g_project_management_issue_design_comments_removed-user: g_project_management_issue_design_comments_removed +g_project_management_issue_designs_added-user: g_project_management_issue_designs_added +g_project_management_issue_designs_modified-user: g_project_management_issue_designs_modified +g_project_management_issue_designs_removed-user: g_project_management_issue_designs_removed +g_project_management_issue_due_date_changed-user: g_project_management_issue_due_date_changed +g_project_management_issue_health_status_changed-user: g_project_management_issue_health_status_changed +g_project_management_issue_iteration_changed-user: g_project_management_issue_iteration_changed +g_project_management_issue_label_changed-user: g_project_management_issue_label_changed +g_project_management_issue_locked-user: g_project_management_issue_locked +g_project_management_issue_made_confidential-user: g_project_management_issue_made_confidential +g_project_management_issue_made_visible-user: g_project_management_issue_made_visible +g_project_management_issue_marked_as_duplicate-user: g_project_management_issue_marked_as_duplicate +g_project_management_issue_milestone_changed-user: g_project_management_issue_milestone_changed +g_project_management_issue_moved-user: g_project_management_issue_moved +g_project_management_issue_promoted_to_epic-user: g_project_management_issue_promoted_to_epic +g_project_management_issue_related-user: g_project_management_issue_related +g_project_management_issue_removed_from_epic-user: g_project_management_issue_removed_from_epic +g_project_management_issue_reopened-user: g_project_management_issue_reopened +g_project_management_issue_time_estimate_changed-user: g_project_management_issue_time_estimate_changed +g_project_management_issue_time_spent_changed-user: g_project_management_issue_time_spent_changed +g_project_management_issue_title_changed-user: g_project_management_issue_title_changed +g_project_management_issue_unlocked-user: g_project_management_issue_unlocked +g_project_management_issue_unrelated-user: g_project_management_issue_unrelated +g_project_management_issue_weight_changed-user: g_project_management_issue_weight_changed +g_project_management_users_awarding_epic_emoji-user: g_project_management_users_awarding_epic_emoji +g_project_management_users_creating_epic_notes-user: g_project_management_users_creating_epic_notes +g_project_management_users_destroying_epic_notes-user: g_project_management_users_destroying_epic_notes +g_project_management_users_epic_issue_added_from_epic-user: g_project_management_users_epic_issue_added_from_epic +g_project_management_users_removing_epic_emoji-user: g_project_management_users_removing_epic_emoji +g_project_management_users_setting_epic_confidential-user: g_project_management_users_setting_epic_confidential +g_project_management_users_setting_epic_due_date_as_fixed-user: g_project_management_users_setting_epic_due_date_as_fixed +g_project_management_users_setting_epic_due_date_as_inherited-user: g_project_management_users_setting_epic_due_date_as_inherited +g_project_management_users_setting_epic_start_date_as_fixed-user: g_project_management_users_setting_epic_start_date_as_fixed +g_project_management_users_setting_epic_start_date_as_inherited-user: g_project_management_users_setting_epic_start_date_as_inherited +g_project_management_users_setting_epic_visible-user: g_project_management_users_setting_epic_visible +g_project_management_users_updating_epic_descriptions-user: g_project_management_users_updating_epic_descriptions +g_project_management_users_updating_epic_notes-user: g_project_management_users_updating_epic_notes +g_project_management_users_updating_epic_parent-user: g_project_management_users_updating_epic_parent +g_project_management_users_updating_epic_titles-user: g_project_management_users_updating_epic_titles +g_project_management_users_updating_fixed_epic_due_date-user: g_project_management_users_updating_fixed_epic_due_date +g_project_management_users_updating_fixed_epic_start_date-user: g_project_management_users_updating_fixed_epic_start_date +geo_secondary_git_op_action-user: geo_secondary_git_op_action +i_analytics_dev_ops_adoption-user: i_analytics_dev_ops_adoption +i_analytics_dev_ops_score-user: i_analytics_dev_ops_score +i_code_review_saved_replies_create-user: i_code_review_saved_replies_create +i_code_review_saved_replies_use-user: i_code_review_saved_replies_use +i_code_review_saved_replies_use_in_mr-user: i_code_review_saved_replies_use_in_mr +i_code_review_saved_replies_use_in_other-user: i_code_review_saved_replies_use_in_other +i_code_review_user_create_mr-user: i_code_review_user_create_mr +i_quickactions_remove_email_multiple-user: i_quickactions_remove_email_multiple +i_quickactions_remove_email_single-user: i_quickactions_remove_email_single +insights_chart_item_clicked-user: insights_chart_item_clicked +insights_issue_chart_item_clicked-user: insights_issue_chart_item_clicked +k8s_api_proxy_requests_unique_users_via_ci_access-user: k8s_api_proxy_requests_unique_users_via_ci_access +k8s_api_proxy_requests_unique_users_via_pat_access-user: k8s_api_proxy_requests_unique_users_via_pat_access +k8s_api_proxy_requests_unique_users_via_user_access-user: k8s_api_proxy_requests_unique_users_via_user_access +p_analytics_ci_cd_deployment_frequency-user: p_analytics_ci_cd_deployment_frequency +p_analytics_ci_cd_lead_time-user: p_analytics_ci_cd_lead_time +p_analytics_ci_cd_pipelines-user: p_analytics_ci_cd_pipelines +project_management_users_checking_epic_task-user: project_management_users_checking_epic_task +project_management_users_unchecking_epic_task-user: project_management_users_unchecking_epic_task +unique_users_visiting_ci_catalog-user: unique_users_visiting_ci_catalog +user_created_custom_dashboard-user: user_created_custom_dashboard +user_created_custom_visualization-user: user_created_custom_visualization +user_edited_cluster_configuration-user: user_edited_cluster_configuration +user_edited_custom_dashboard-user: user_edited_custom_dashboard +user_viewed_cluster_configuration-user: user_viewed_cluster_configuration +user_viewed_custom_dashboard-user: user_viewed_custom_dashboard +user_viewed_dashboard_designer-user: user_viewed_dashboard_designer +user_viewed_dashboard_list-user: user_viewed_dashboard_list +user_viewed_instrumentation_directions-user: user_viewed_instrumentation_directions +user_viewed_visualization_designer-user: user_viewed_visualization_designer +user_visited_dashboard-user: user_visited_dashboard +value_streams_dashboard_change_failure_rate_link_clicked-user: value_streams_dashboard_change_failure_rate_link_clicked +value_streams_dashboard_cycle_time_link_clicked-user: value_streams_dashboard_cycle_time_link_clicked +value_streams_dashboard_deployment_frequency_link_clicked-user: value_streams_dashboard_deployment_frequency_link_clicked +value_streams_dashboard_deploys_link_clicked-user: value_streams_dashboard_deploys_link_clicked +value_streams_dashboard_issues_completed_link_clicked-user: value_streams_dashboard_issues_completed_link_clicked +value_streams_dashboard_issues_link_clicked-user: value_streams_dashboard_issues_link_clicked +value_streams_dashboard_lead_time_for_changes_link_clicked-user: value_streams_dashboard_lead_time_for_changes_link_clicked +value_streams_dashboard_lead_time_link_clicked-user: value_streams_dashboard_lead_time_link_clicked +value_streams_dashboard_merge_request_throughput_link_clicked-user: value_streams_dashboard_merge_request_throughput_link_clicked +value_streams_dashboard_metric_link_clicked-user: value_streams_dashboard_metric_link_clicked +value_streams_dashboard_time_to_restore_service_link_clicked-user: value_streams_dashboard_time_to_restore_service_link_clicked +value_streams_dashboard_vulnerability_critical_link_clicked-user: value_streams_dashboard_vulnerability_critical_link_clicked +value_streams_dashboard_vulnerability_high_link_clicked-user: value_streams_dashboard_vulnerability_high_link_clicked diff --git a/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb b/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb index da8098bfee1dbc8172746887544a48c5b53a25a8..5077c3532effa9269f9a0bc4ab9ea6ed2946703c 100644 --- a/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb +++ b/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb @@ -62,15 +62,15 @@ describe 'known_events' do let(:weekly_event) { 'g_analytics_contribution' } - let(:daily_event) { 'g_analytics_search' } + let(:daily_event) { 'g_analytics_issues' } let(:analytics_slot_event) { 'g_analytics_contribution' } let(:compliance_slot_event) { 'g_compliance_dashboard' } - let(:category_analytics_event) { 'g_analytics_search' } + let(:category_analytics_event) { 'g_analytics_issues' } let(:category_productivity_event) { 'g_analytics_productivity' } let(:no_slot) { 'no_slot' } let(:different_aggregation) { 'different_aggregation' } let(:custom_daily_event) { 'g_analytics_custom' } - + let(:event_overridden_for_user) { 'user_created_custom_dashboard' } let(:global_category) { 'global' } let(:compliance_category) { 'compliance' } let(:productivity_category) { 'productivity' } @@ -84,7 +84,8 @@ { name: category_productivity_event }, { name: compliance_slot_event }, { name: no_slot }, - { name: different_aggregation } + { name: different_aggregation }, + { name: event_overridden_for_user } ].map(&:with_indifferent_access) end @@ -216,6 +217,97 @@ end end end + + describe "property_name" do + before do + stub_feature_flags(redis_hll_property_name_tracking: property_name_flag_enabled) + end + + context "with enabled feature flag" do + let(:property_name_flag_enabled) { true } + + context "with a property_name for an overridden event" do + context "with a property_name sent as a symbol" do + it "tracks the events using the Redis key override" do + expected_key = "{hll_counters}_#{event_overridden_for_user}-2020-23" + expect(Gitlab::Redis::HLL).to receive(:add).with(hash_including(key: expected_key)) + + described_class.track_event(event_overridden_for_user, values: entity1, property_name: :user) + end + end + + context "with a property_name sent in string format" do + it "tracks the events using the Redis key override" do + expected_key = "{hll_counters}_#{event_overridden_for_user}-2020-23" + expect(Gitlab::Redis::HLL).to receive(:add).with(hash_including(key: expected_key)) + + described_class.track_event(event_overridden_for_user, values: entity1, property_name: 'user.id') + end + end + end + + context "with a property_name for an overridden event that doesn't include this property_name" do + it "tracks the events using a Redis key with the property_name" do + expected_key = "{hll_counters}_#{no_slot}-user-2020-23" + expect(Gitlab::Redis::HLL).to receive(:add).with(hash_including(key: expected_key)) + + described_class.track_event(no_slot, values: entity1, property_name: 'user') + end + end + + context "with a property_name for a new event" do + it "tracks the events using a Redis key with the property_name" do + expected_key = "{hll_counters}_#{no_slot}-project-2020-23" + expect(Gitlab::Redis::HLL).to receive(:add).with(hash_including(key: expected_key)) + + described_class.track_event(no_slot, values: entity1, property_name: 'project') + end + end + + context "with no property_name for an overridden event" do + it "tracks the events using a Redis key with no property_name" do + expected_key = "{hll_counters}_#{event_overridden_for_user}-2020-23" + expect(Gitlab::Redis::HLL).to receive(:add).with(hash_including(key: expected_key)) + + described_class.track_event(event_overridden_for_user, values: entity1) + end + end + + context "with no property_name for a new event" do + it "tracks the events using a Redis key with no property_name" do + expected_key = "{hll_counters}_#{no_slot}-2020-23" + expect(Gitlab::Redis::HLL).to receive(:add).with(hash_including(key: expected_key)) + + described_class.track_event(no_slot, values: entity1) + end + end + end + + context "with disabled feature flag" do + let(:property_name_flag_enabled) { false } + + it "uses old Redis key for overridden events" do + expected_key = "{hll_counters}_#{event_overridden_for_user}-2020-23" + expect(Gitlab::Redis::HLL).to receive(:add).with(hash_including(key: expected_key)) + + described_class.track_event(event_overridden_for_user, values: entity1, property_name: 'user') + end + + it "uses old Redis key for new events" do + expected_key = "{hll_counters}_#{no_slot}-2020-23" + expect(Gitlab::Redis::HLL).to receive(:add).with(hash_including(key: expected_key)) + + described_class.track_event(no_slot, values: entity1, property_name: 'project') + end + + it "uses old Redis key for new events when no property name sent" do + expected_key = "{hll_counters}_#{no_slot}-2020-23" + expect(Gitlab::Redis::HLL).to receive(:add).with(hash_including(key: expected_key)) + + described_class.track_event(no_slot, values: entity1) + end + end + end end describe '.unique_events' do @@ -227,7 +319,7 @@ # Events last week described_class.track_event(weekly_event, values: entity1, time: 2.days.ago) described_class.track_event(weekly_event, values: entity1, time: 2.days.ago) - described_class.track_event(no_slot, values: entity1, time: 2.days.ago) + described_class.track_event(no_slot, values: entity1, property_name: 'user.id', time: 2.days.ago) # Events 2 weeks ago described_class.track_event(weekly_event, values: entity1, time: 2.weeks.ago) @@ -274,7 +366,7 @@ end context 'when no slot is set' do - it { expect(described_class.unique_events(event_names: [no_slot], start_date: 7.days.ago, end_date: Date.current)).to eq(1) } + it { expect(described_class.unique_events(event_names: [no_slot], property_name: 'user.id', start_date: 7.days.ago, end_date: Date.current)).to eq(1) } end context 'when data crosses into new year' do @@ -283,6 +375,97 @@ .not_to raise_error end end + + describe "property_names" do + before do + stub_feature_flags(redis_hll_property_name_tracking: property_name_flag_enabled) + end + + context "with enabled feature flag" do + let(:property_name_flag_enabled) { true } + + context "with a property_name for an overridden event" do + context "with a property_name sent as a symbol" do + it "tracks the events using the Redis key override" do + expected_key = "{hll_counters}_#{event_overridden_for_user}-2020-22" + expect(Gitlab::Redis::HLL).to receive(:count).with(keys: [expected_key]) + + described_class.unique_events(event_names: [event_overridden_for_user], property_name: :user, start_date: 7.days.ago, end_date: Date.current) + end + end + + context "with a property_name sent in string format" do + it "tracks the events using the Redis key override" do + expected_key = "{hll_counters}_#{event_overridden_for_user}-2020-22" + expect(Gitlab::Redis::HLL).to receive(:count).with(keys: [expected_key]) + + described_class.unique_events(event_names: [event_overridden_for_user], property_name: 'user.id', start_date: 7.days.ago, end_date: Date.current) + end + end + end + + context "with a property_name for an overridden event that doesn't include this property_name" do + it "tracks the events using a Redis key with the property_name" do + expected_key = "{hll_counters}_#{no_slot}-user-2020-22" + expect(Gitlab::Redis::HLL).to receive(:count).with(keys: [expected_key]) + + described_class.unique_events(event_names: [no_slot], property_name: 'user', start_date: 7.days.ago, end_date: Date.current) + end + end + + context "with a property_name for a new event" do + it "tracks the events using a Redis key with the property_name" do + expected_key = "{hll_counters}_#{no_slot}-project-2020-22" + expect(Gitlab::Redis::HLL).to receive(:count).with(keys: [expected_key]) + + described_class.unique_events(event_names: [no_slot], property_name: 'project', start_date: 7.days.ago, end_date: Date.current) + end + end + + context "with no property_name for a overridden event" do + it "tracks the events using a Redis key with no property_name" do + expected_key = "{hll_counters}_#{event_overridden_for_user}-2020-22" + expect(Gitlab::Redis::HLL).to receive(:count).with(keys: [expected_key]) + + described_class.unique_events(event_names: [event_overridden_for_user], start_date: 7.days.ago, end_date: Date.current) + end + end + + context "with no property_name for a new event" do + it "tracks the events using a Redis key with no property_name" do + expected_key = "{hll_counters}_#{no_slot}-2020-22" + expect(Gitlab::Redis::HLL).to receive(:count).with(keys: [expected_key]) + + described_class.unique_events(event_names: [no_slot], start_date: 7.days.ago, end_date: Date.current) + end + end + end + + context "with disabled feature flag" do + let(:property_name_flag_enabled) { false } + + it "uses old Redis key for overridden events" do + expected_key = "{hll_counters}_#{event_overridden_for_user}-2020-22" + expect(Gitlab::Redis::HLL).to receive(:count).with(keys: [expected_key]) + + described_class.unique_events(event_names: [event_overridden_for_user], property_name: 'user', start_date: 7.days.ago, end_date: Date.current) + end + + it "uses old Redis key for new events" do + expected_key = "{hll_counters}_#{no_slot}-2020-22" + expect(Gitlab::Redis::HLL).to receive(:count).with(keys: [expected_key]) + + described_class.unique_events(event_names: [no_slot], property_name: 'project', start_date: 7.days.ago, end_date: Date.current) + end + + it "uses old Redis key for new events when no property name sent" do + expected_key = "{hll_counters}_#{no_slot}-2020-22" + expect(Gitlab::Redis::HLL).to receive(:count).with(keys: [expected_key]) + + described_class.unique_events(event_names: [no_slot], start_date: 7.days.ago, end_date: Date.current) + end + end + end end describe 'key overrides file' do @@ -341,43 +524,43 @@ let(:time_range) { { start_date: 7.days.ago, end_date: DateTime.current } } let(:known_events) do [ - { name: 'event1_slot' }, - { name: 'event2_slot' }, - { name: 'event3_slot' }, - { name: 'event5_slot' }, - { name: 'event4' } + { name: 'g_compliance_dashboard' }, + { name: 'g_project_management_epic_created' }, + { name: 'g_project_management_epic_closed' }, + { name: 'g_project_management_epic_reopened' }, + { name: 'g_project_management_epic_issue_added' } ].map(&:with_indifferent_access) end before do allow(described_class).to receive(:known_events).and_return(known_events) - described_class.track_event('event1_slot', values: entity1, time: 2.days.ago) - described_class.track_event('event1_slot', values: entity2, time: 2.days.ago) - described_class.track_event('event1_slot', values: entity3, time: 2.days.ago) - described_class.track_event('event2_slot', values: entity1, time: 2.days.ago) - described_class.track_event('event2_slot', values: entity2, time: 3.days.ago) - described_class.track_event('event2_slot', values: entity3, time: 3.days.ago) - described_class.track_event('event3_slot', values: entity1, time: 3.days.ago) - described_class.track_event('event3_slot', values: entity2, time: 3.days.ago) - described_class.track_event('event5_slot', values: entity2, time: 3.days.ago) + described_class.track_event('g_compliance_dashboard', values: entity1, time: 2.days.ago) + described_class.track_event('g_compliance_dashboard', values: entity2, time: 2.days.ago) + described_class.track_event('g_compliance_dashboard', values: entity3, time: 2.days.ago) + described_class.track_event('g_project_management_epic_created', values: entity1, time: 2.days.ago) + described_class.track_event('g_project_management_epic_created', values: entity2, time: 3.days.ago) + described_class.track_event('g_project_management_epic_created', values: entity3, time: 3.days.ago) + described_class.track_event('g_project_management_epic_closed', values: entity1, time: 3.days.ago) + described_class.track_event('g_project_management_epic_closed', values: entity2, time: 3.days.ago) + described_class.track_event('g_project_management_epic_reopened', values: entity2, time: 3.days.ago) # events out of time scope - described_class.track_event('event2_slot', values: entity4, time: 8.days.ago) + described_class.track_event('g_project_management_epic_created', values: entity4, time: 8.days.ago) # events in different slots - described_class.track_event('event4', values: entity1, time: 2.days.ago) - described_class.track_event('event4', values: entity2, time: 2.days.ago) + described_class.track_event('g_project_management_epic_issue_added', values: entity1, time: 2.days.ago) + described_class.track_event('g_project_management_epic_issue_added', values: entity2, time: 2.days.ago) end it 'calculates union of given events', :aggregate_failures do - expect(described_class.calculate_events_union(**time_range.merge(event_names: %w[event4]))).to eq 2 - expect(described_class.calculate_events_union(**time_range.merge(event_names: %w[event1_slot event2_slot event3_slot]))).to eq 3 + expect(described_class.calculate_events_union(**time_range.merge(event_names: %w[g_project_management_epic_issue_added]))).to eq 2 + expect(described_class.calculate_events_union(**time_range.merge(event_names: %w[g_compliance_dashboard g_project_management_epic_created g_project_management_epic_closed]))).to eq 3 end it 'returns 0 if there are no keys for given events' do expect(Gitlab::Redis::HLL).not_to receive(:count) - expect(described_class.calculate_events_union(event_names: %w[event1_slot event2_slot event3_slot], start_date: Date.current, end_date: 4.weeks.ago)).to eq(-1) + expect(described_class.calculate_events_union(event_names: %w[g_compliance_dashboard g_project_management_epic_created g_project_management_epic_closed], start_date: Date.current, end_date: 4.weeks.ago)).to eq(-1) end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index d08f6ef9d0dd0eefabb88e7e848d7d659e50b905..2add01807cb7f802b3f68e7ce9ed46a13f826342 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -330,6 +330,9 @@ # Postgres is the primary data source, and ClickHouse only when enabled in certain cases. stub_feature_flags(clickhouse_data_collection: false) + + # This is going to be removed with https://gitlab.com/gitlab-org/gitlab/-/issues/432866 + stub_feature_flags(redis_hll_property_name_tracking: false) else unstub_all_feature_flags end