From 11a483649e2bbcb2ee293fd189473a048d3e3f35 Mon Sep 17 00:00:00 2001
From: Zeger-Jan van de Weg <git@zjvandeweg.nl>
Date: Tue, 27 Mar 2018 11:48:11 +0200
Subject: [PATCH] Test if remote repository exists before cloning
When a repository does not exist on a remote, Gitaly won't be able to
clone it. This is correct behaviour, but from the clients perspective a
change in behaviour.
This change implements the client side changes that allows Gitaly to
execute a `git ls-remote <remote-url> HEAD`. This way the client has no
need to shell out to Git.
In the situation where multiple Gitalies are available, one is chosen at
random.
This commit closes https://gitlab.com/gitlab-org/gitlab-ce/issues/43929,
while its also a part of https://gitlab.com/gitlab-org/gitaly/issues/1084
---
GITALY_SERVER_VERSION | 2 +-
Gemfile | 2 +-
Gemfile.lock | 4 ++--
changelogs/unreleased/zj-remote-repo-exists.yml | 5 +++++
lib/gitlab/gitaly_client.rb | 4 ++++
lib/gitlab/gitaly_client/remote_service.rb | 11 +++++++++++
.../github_import/importer/repository_importer.rb | 8 ++++++--
spec/lib/gitlab/gitaly_client/remote_service_spec.rb | 10 ++++++++++
.../importer/repository_importer_spec.rb | 6 +++++-
9 files changed, 45 insertions(+), 7 deletions(-)
create mode 100644 changelogs/unreleased/zj-remote-repo-exists.yml
diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION
index 8f63f4f9a10c8..36545ad338e5f 100644
--- a/GITALY_SERVER_VERSION
+++ b/GITALY_SERVER_VERSION
@@ -1 +1 @@
-0.91.0
+0.92.0
diff --git a/Gemfile b/Gemfile
index a670d86104cfb..d6f4615643c3d 100644
--- a/Gemfile
+++ b/Gemfile
@@ -420,7 +420,7 @@ group :ed25519 do
end
# Gitaly GRPC client
-gem 'gitaly-proto', '~> 0.88.0', require: 'gitaly'
+gem 'gitaly-proto', '~> 0.91.0', require: 'gitaly'
gem 'grpc', '~> 1.10.0'
# Locked until https://github.com/google/protobuf/issues/4210 is closed
diff --git a/Gemfile.lock b/Gemfile.lock
index 61107a2130b55..f15328cfc0fd2 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -289,7 +289,7 @@ GEM
po_to_json (>= 1.0.0)
rails (>= 3.2.0)
gherkin-ruby (0.3.2)
- gitaly-proto (0.88.0)
+ gitaly-proto (0.91.0)
google-protobuf (~> 3.1)
grpc (~> 1.0)
github-linguist (5.3.3)
@@ -1058,7 +1058,7 @@ DEPENDENCIES
gettext (~> 3.2.2)
gettext_i18n_rails (~> 1.8.0)
gettext_i18n_rails_js (~> 1.3)
- gitaly-proto (~> 0.88.0)
+ gitaly-proto (~> 0.91.0)
github-linguist (~> 5.3.3)
gitlab-flowdock-git-hook (~> 1.0.1)
gitlab-markup (~> 1.6.2)
diff --git a/changelogs/unreleased/zj-remote-repo-exists.yml b/changelogs/unreleased/zj-remote-repo-exists.yml
new file mode 100644
index 0000000000000..f024b83159bba
--- /dev/null
+++ b/changelogs/unreleased/zj-remote-repo-exists.yml
@@ -0,0 +1,5 @@
+---
+title: Test if remote repository exists when importing wikis
+merge_request:
+author:
+type: fixed
diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index 8ca30ffc2328c..0abae70c443cb 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -83,6 +83,10 @@ def self.clear_stubs!
end
end
+ def self.random_storage
+ Gitlab.config.repositories.storages.keys.sample
+ end
+
def self.address(storage)
params = Gitlab.config.repositories.storages[storage]
raise "storage not found: #{storage.inspect}" if params.nil?
diff --git a/lib/gitlab/gitaly_client/remote_service.rb b/lib/gitlab/gitaly_client/remote_service.rb
index 58c356edfd1e5..f2d699d9dfb1e 100644
--- a/lib/gitlab/gitaly_client/remote_service.rb
+++ b/lib/gitlab/gitaly_client/remote_service.rb
@@ -3,6 +3,17 @@ module GitalyClient
class RemoteService
MAX_MSG_SIZE = 128.kilobytes.freeze
+ def self.exists?(remote_url)
+ request = Gitaly::FindRemoteRepositoryRequest.new(remote: remote_url)
+
+ response = GitalyClient.call(GitalyClient.random_storage,
+ :remote_service,
+ :find_remote_repository, request,
+ timeout: GitalyClient.medium_timeout)
+
+ response.exists
+ end
+
def initialize(repository)
@repository = repository
@gitaly_repo = repository.gitaly_repository
diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb
index ab0b751fe242e..b1b283e98b5bf 100644
--- a/lib/gitlab/github_import/importer/repository_importer.rb
+++ b/lib/gitlab/github_import/importer/repository_importer.rb
@@ -16,7 +16,8 @@ def initialize(project, client)
# Returns true if we should import the wiki for the project.
def import_wiki?
client.repository(project.import_source)&.has_wiki &&
- !project.wiki_repository_exists?
+ !project.wiki_repository_exists? &&
+ Gitlab::GitalyClient::RemoteService.exists?(wiki_url)
end
# Imports the repository data.
@@ -55,7 +56,6 @@ def import_repository
def import_wiki_repository
wiki_path = "#{project.disk_path}.wiki"
- wiki_url = project.import_url.sub(/\.git\z/, '.wiki.git')
storage_path = project.repository_storage_path
gitlab_shell.import_repository(storage_path, wiki_path, wiki_url)
@@ -70,6 +70,10 @@ def import_wiki_repository
end
end
+ def wiki_url
+ project.import_url.sub(/\.git\z/, '.wiki.git')
+ end
+
def update_clone_time
project.update_column(:last_repository_updated_at, Time.zone.now)
end
diff --git a/spec/lib/gitlab/gitaly_client/remote_service_spec.rb b/spec/lib/gitlab/gitaly_client/remote_service_spec.rb
index 872377c93d8ab..f03c7e3f04b42 100644
--- a/spec/lib/gitlab/gitaly_client/remote_service_spec.rb
+++ b/spec/lib/gitlab/gitaly_client/remote_service_spec.rb
@@ -58,4 +58,14 @@
client.update_remote_mirror(ref_name, only_branches_matching)
end
end
+
+ describe '.exists?' do
+ context "when the remote doesn't exist" do
+ let(:url) { 'https://gitlab.com/gitlab-org/ik-besta-niet-of-ik-word-geplaagd.git' }
+
+ it 'returns false' do
+ expect(described_class.exists?(url)).to be(false)
+ end
+ end
+ end
end
diff --git a/spec/lib/gitlab/github_import/importer/repository_importer_spec.rb b/spec/lib/gitlab/github_import/importer/repository_importer_spec.rb
index 5bedfc79dd31a..1f0f1fdd7da18 100644
--- a/spec/lib/gitlab/github_import/importer/repository_importer_spec.rb
+++ b/spec/lib/gitlab/github_import/importer/repository_importer_spec.rb
@@ -38,8 +38,12 @@
expect(project)
.to receive(:wiki_repository_exists?)
.and_return(false)
+ expect(Gitlab::GitalyClient::RemoteService)
+ .to receive(:exists?)
+ .with("foo.wiki.git")
+ .and_return(true)
- expect(importer.import_wiki?).to eq(true)
+ expect(importer.import_wiki?).to be(true)
end
it 'returns false if the GitHub wiki is disabled' do
--
GitLab