Merge branch '292861-github-importer-automatically-refresh-import-status' into 'master'

Expire relevant ETag caches for importers

See merge request gitlab-org/gitlab!83849

app/controllers/import/base_controller.rb

@@ -70,7 +70,7 @@ def serialized_imported_projects
   end
 
   def already_added_projects
-    @already_added_projects ||= filtered(find_already_added_projects(provider_name))
+    @already_added_projects ||= find_already_added_projects(provider_name)
   end
 
   # rubocop: disable CodeReuse/ActiveRecord

app/models/project_import_state.rb

@@ -6,6 +6,8 @@ class ProjectImportState < ApplicationRecord
 
   self.table_name = "project_mirror_data"
 
+  after_commit :expire_etag_cache
+
   belongs_to :project, inverse_of: :import_state
 
   validates :project, presence: true
@@ -76,6 +78,23 @@ class ProjectImportState < ApplicationRecord
     end
   end
 
+  def expire_etag_cache
+    if realtime_changes_path
+      Gitlab::EtagCaching::Store.new.tap do |store|
+        store.touch(realtime_changes_path)
+      rescue Gitlab::EtagCaching::Store::InvalidKeyError
+        # no-op: not every realtime changes endpoint is using etag caching
+      end
+    end
+  end
+
+  def realtime_changes_path
+    Gitlab::Routing.url_helpers.polymorphic_path([:realtime_changes_import, project.import_type.to_sym], format: :json)
+  rescue NoMethodError
+    # polymorphic_path throws NoMethodError when no such path exists
+    nil
+  end
+
   def relation_hard_failures(limit:)
     project.import_failures.hard_failures_by_correlation_id(correlation_id).limit(limit)
   end

ee/spec/features/registrations/combined_registration_spec.rb

@@ -8,8 +8,8 @@
   let(:experiments) { {} }
 
   before do
-    # https://gitlab.com/gitlab-org/gitlab/-/issues/338737
-    stub_const('Gitlab::QueryLimiting::Transaction::THRESHOLD', 250)
+    # https://gitlab.com/gitlab-org/gitlab/-/issues/340302
+    stub_const('Gitlab::QueryLimiting::Transaction::THRESHOLD', 270)
     stub_experiments(experiments)
     allow(Gitlab).to receive(:com?).and_return(true)
     sign_in(user)

lib/gitlab/github_import/object_counter.rb

@@ -24,6 +24,8 @@ def increment(project, object_type, operation, value: 1)
 
           increment_project_counter(project, object_type, operation, integer)
           increment_global_counter(object_type, operation, integer)
+
+          project.import_state&.expire_etag_cache
         end
 
         def summary(project)

spec/controllers/import/github_controller_spec.rb

@@ -258,7 +258,9 @@
 
         context 'when user input contains colons and spaces' do
           before do
-            allow(controller).to receive(:client_repos).and_return([])
+            allow_next_instance_of(Gitlab::GithubImport::Client) do |client|
+              allow(client).to receive(:search_repos_by_name).and_return(items: [])
+            end
           end
 
           it 'sanitizes user input' do

spec/lib/gitlab/github_import/object_counter_spec.rb

@@ -3,7 +3,7 @@
 require 'spec_helper'
 
 RSpec.describe Gitlab::GithubImport::ObjectCounter, :clean_gitlab_redis_cache do
-  let_it_be(:project) { create(:project) }
+  let_it_be(:project) { create(:project, :import_started, import_type: 'github') }
 
   it 'validates the operation being incremented' do
     expect { described_class.increment(project, :issue, :unknown) }
@@ -49,4 +49,12 @@
       'imported' => {}
     })
   end
+
+  it 'expires etag cache of relevant realtime change endpoints on increment' do
+    expect_next_instance_of(Gitlab::EtagCaching::Store) do |instance|
+      expect(instance).to receive(:touch).with(Gitlab::Routing.url_helpers.realtime_changes_import_github_path(format: :json))
+    end
+
+    described_class.increment(project, :issue, :fetched)
+  end
 end

spec/models/project_import_state_spec.rb

@@ -101,6 +101,34 @@
     end
   end
 
+  describe '#expire_etag_cache' do
+    context 'when project import type has realtime changes endpoint' do
+      before do
+        import_state.project.import_type = 'github'
+      end
+
+      it 'expires revelant etag cache' do
+        expect_next_instance_of(Gitlab::EtagCaching::Store) do |instance|
+          expect(instance).to receive(:touch).with(Gitlab::Routing.url_helpers.realtime_changes_import_github_path(format: :json))
+        end
+
+        subject.expire_etag_cache
+      end
+    end
+
+    context 'when project import type does not have realtime changes endpoint' do
+      before do
+        import_state.project.import_type = 'jira'
+      end
+
+      it 'does not touch etag caches' do
+        expect(Gitlab::EtagCaching::Store).not_to receive(:new)
+
+        subject.expire_etag_cache
+      end
+    end
+  end
+
   describe 'import state transitions' do
     context 'state transition: [:started] => [:finished]' do
       let(:after_import_service) { spy(:after_import_service) }
@@ -178,4 +206,20 @@
       end
     end
   end
+
+  describe 'callbacks' do
+    context 'after_commit :expire_etag_cache' do
+      before do
+        import_state.project.import_type = 'github'
+      end
+
+      it 'expires etag cache' do
+        expect_next_instance_of(Gitlab::EtagCaching::Store) do |instance|
+          expect(instance).to receive(:touch).with(Gitlab::Routing.url_helpers.realtime_changes_import_github_path(format: :json))
+        end
+
+        subject.save!
+      end
+    end
+  end
 end

spec/workers/gitlab/github_import/import_diff_note_worker_spec.rb

@@ -7,7 +7,7 @@
 
   describe '#import' do
     it 'imports a diff note' do
-      project = double(:project, full_path: 'foo/bar', id: 1)
+      project = double(:project, full_path: 'foo/bar', id: 1, import_state: nil)
       client = double(:client)
       importer = double(:importer)
       hash = {

spec/workers/gitlab/github_import/import_issue_worker_spec.rb

@@ -7,7 +7,7 @@
 
   describe '#import' do
     it 'imports an issue' do
-      project = double(:project, full_path: 'foo/bar', id: 1)
+      project = double(:project, full_path: 'foo/bar', id: 1, import_state: nil)
       client = double(:client)
       importer = double(:importer)
       hash = {

spec/workers/gitlab/github_import/import_note_worker_spec.rb

@@ -7,7 +7,7 @@
 
   describe '#import' do
     it 'imports a note' do
-      project = double(:project, full_path: 'foo/bar', id: 1)
+      project = double(:project, full_path: 'foo/bar', id: 1, import_state: nil)
       client = double(:client)
       importer = double(:importer)
       hash = {

spec/workers/gitlab/github_import/import_pull_request_worker_spec.rb

@@ -7,7 +7,7 @@
 
   describe '#import' do
     it 'imports a pull request' do
-      project = double(:project, full_path: 'foo/bar', id: 1)
+      project = double(:project, full_path: 'foo/bar', id: 1, import_state: nil)
       client = double(:client)
       importer = double(:importer)
       hash = {