diff --git a/changelogs/unreleased/backstage-gb-stages-position-migration-clean-up.yml b/changelogs/unreleased/backstage-gb-stages-position-migration-clean-up.yml
new file mode 100644
index 0000000000000000000000000000000000000000..d2ada88870b13291dc931d96acdb4a09456a76e2
--- /dev/null
+++ b/changelogs/unreleased/backstage-gb-stages-position-migration-clean-up.yml
@@ -0,0 +1,5 @@
+---
+title: Fully migrate pipeline stages position
+merge_request: 19369
+author:
+type: performance
diff --git a/db/post_migrate/20180604123514_cleanup_stages_position_migration.rb b/db/post_migrate/20180604123514_cleanup_stages_position_migration.rb
new file mode 100644
index 0000000000000000000000000000000000000000..73c23dffca0ebc1da3c0ced3a23f5bee6b233660
--- /dev/null
+++ b/db/post_migrate/20180604123514_cleanup_stages_position_migration.rb
@@ -0,0 +1,43 @@
+class CleanupStagesPositionMigration < ActiveRecord::Migration
+ include Gitlab::Database::MigrationHelpers
+
+ DOWNTIME = false
+ TMP_INDEX_NAME = 'tmp_id_stage_position_partial_null_index'.freeze
+
+ disable_ddl_transaction!
+
+ class Stages < ActiveRecord::Base
+ include EachBatch
+ self.table_name = 'ci_stages'
+ end
+
+ def up
+ disable_statement_timeout
+
+ Gitlab::BackgroundMigration.steal('MigrateStageIndex')
+
+ unless index_exists_by_name?(:ci_stages, TMP_INDEX_NAME)
+ add_concurrent_index(:ci_stages, :id, where: 'position IS NULL', name: TMP_INDEX_NAME)
+ end
+
+ migratable = <<~SQL
+ position IS NULL AND EXISTS (
+ SELECT 1 FROM ci_builds WHERE stage_id = ci_stages.id AND stage_idx IS NOT NULL
+ )
+ SQL
+
+ Stages.where(migratable).each_batch(of: 1000) do |batch|
+ batch.pluck(:id).each do |stage|
+ Gitlab::BackgroundMigration::MigrateStageIndex.new.perform(stage, stage)
+ end
+ end
+
+ remove_concurrent_index_by_name(:ci_stages, TMP_INDEX_NAME)
+ end
+
+ def down
+ if index_exists_by_name?(:ci_stages, TMP_INDEX_NAME)
+ remove_concurrent_index_by_name(:ci_stages, TMP_INDEX_NAME)
+ end
+ end
+end
diff --git a/doc/user/project/settings/import_export.md b/doc/user/project/settings/import_export.md
index 9034a9b51792f19706da3df7341c2638ec244a99..f94671fcf875432823f2fe3f1b1cc7c7ffc2706c 100644
--- a/doc/user/project/settings/import_export.md
+++ b/doc/user/project/settings/import_export.md
@@ -32,7 +32,8 @@ with all their related data and be moved into a new GitLab instance.
| GitLab version | Import/Export version |
| ---------------- | --------------------- |
-| 10.8 to current | 0.2.3 |
+| 11.1 to current | 0.2.4 |
+| 10.8 | 0.2.3 |
| 10.4 | 0.2.2 |
| 10.3 | 0.2.1 |
| 10.0 | 0.2.0 |
diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb
index b713fa7e1cd12f3f0e495781ce83089b64b48fec..53fe2f8e4361529b40cf6bee65d7fc4309d14641 100644
--- a/lib/gitlab/import_export.rb
+++ b/lib/gitlab/import_export.rb
@@ -3,7 +3,7 @@ module ImportExport
extend self
# For every version update, the version history in import_export.md has to be kept up to date.
- VERSION = '0.2.3'.freeze
+ VERSION = '0.2.4'.freeze
FILENAME_LIMIT = 50
def export_path(relative_path:)
diff --git a/lib/tasks/gitlab/import_export.rake b/lib/tasks/gitlab/import_export.rake
index 44074397c05ef963ac0185d3b848d75153b47cb5..900dbf7be2421f84a9429444270fcc3f3363bfe2 100644
--- a/lib/tasks/gitlab/import_export.rake
+++ b/lib/tasks/gitlab/import_export.rake
@@ -10,15 +10,22 @@ namespace :gitlab do
puts YAML.load_file(Gitlab::ImportExport.config_file)['project_tree'].to_yaml(SortKeys: true)
end
- desc 'GitLab | Bumps the Import/Export version for test_project_export.tar.gz'
- task bump_test_version: :environment do
- Dir.mktmpdir do |tmp_dir|
- system("tar -zxf spec/features/projects/import_export/test_project_export.tar.gz -C #{tmp_dir} > /dev/null")
- File.write(File.join(tmp_dir, 'VERSION'), Gitlab::ImportExport.version, mode: 'w')
- system("tar -zcvf spec/features/projects/import_export/test_project_export.tar.gz -C #{tmp_dir} . > /dev/null")
+ desc 'GitLab | Bumps the Import/Export version in fixtures and project templates'
+ task bump_version: :environment do
+ archives = Dir['vendor/project_templates/*.tar.gz']
+ archives.push('spec/features/projects/import_export/test_project_export.tar.gz')
+
+ archives.each do |archive|
+ raise ArgumentError unless File.exist?(archive)
+
+ Dir.mktmpdir do |tmp_dir|
+ system("tar -zxf #{archive} -C #{tmp_dir} > /dev/null")
+ File.write(File.join(tmp_dir, 'VERSION'), Gitlab::ImportExport.version, mode: 'w')
+ system("tar -zcvf #{archive} -C #{tmp_dir} . > /dev/null")
+ end
end
- puts "Updated to #{Gitlab::ImportExport.version}"
+ puts "Updated #{archives} to #{Gitlab::ImportExport.version}."
end
end
end
diff --git a/spec/features/projects/import_export/test_project_export.tar.gz b/spec/features/projects/import_export/test_project_export.tar.gz
index 72ab2d71f355c1079020afa149105daf2533c199..ceba4dfec5707072e7c031612c15f098a41ba594 100644
Binary files a/spec/features/projects/import_export/test_project_export.tar.gz and b/spec/features/projects/import_export/test_project_export.tar.gz differ
diff --git a/spec/fixtures/exported-project.gz b/spec/fixtures/exported-project.gz
deleted file mode 100644
index bef7e2ff8ee9a6cfa6ddd738ae3ad089a8682b90..0000000000000000000000000000000000000000
Binary files a/spec/fixtures/exported-project.gz and /dev/null differ
diff --git a/spec/lib/gitlab/import_export/importer_spec.rb b/spec/lib/gitlab/import_export/importer_spec.rb
index 991e354f499f7a218e5c0219c04f0c7349d46d52..c074e61da26f8713cb5ce6b4967d436b6b91898e 100644
--- a/spec/lib/gitlab/import_export/importer_spec.rb
+++ b/spec/lib/gitlab/import_export/importer_spec.rb
@@ -4,14 +4,14 @@
let(:user) { create(:user) }
let(:test_path) { "#{Dir.tmpdir}/importer_spec" }
let(:shared) { project.import_export_shared }
- let(:project) { create(:project, import_source: File.join(test_path, 'exported-project.gz')) }
+ let(:project) { create(:project, import_source: File.join(test_path, 'test_project_export.tar.gz')) }
subject(:importer) { described_class.new(project) }
before do
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(test_path)
FileUtils.mkdir_p(shared.export_path)
- FileUtils.cp(Rails.root.join('spec', 'fixtures', 'exported-project.gz'), test_path)
+ FileUtils.cp(Rails.root.join('spec/features/projects/import_export/test_project_export.tar.gz'), test_path)
allow(subject).to receive(:remove_import_file)
end
diff --git a/spec/migrations/cleanup_stages_position_migration_spec.rb b/spec/migrations/cleanup_stages_position_migration_spec.rb
new file mode 100644
index 0000000000000000000000000000000000000000..dde5a7774878a3c195aa90d9c03ed0417fa10837
--- /dev/null
+++ b/spec/migrations/cleanup_stages_position_migration_spec.rb
@@ -0,0 +1,67 @@
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20180604123514_cleanup_stages_position_migration.rb')
+
+describe CleanupStagesPositionMigration, :migration, :sidekiq, :redis do
+ let(:migration) { spy('migration') }
+
+ before do
+ allow(Gitlab::BackgroundMigration::MigrateStageIndex)
+ .to receive(:new).and_return(migration)
+ end
+
+ context 'when there are pending background migrations' do
+ it 'processes pending jobs synchronously' do
+ Sidekiq::Testing.disable! do
+ BackgroundMigrationWorker
+ .perform_in(2.minutes, 'MigrateStageIndex', [1, 1])
+ BackgroundMigrationWorker
+ .perform_async('MigrateStageIndex', [1, 1])
+
+ migrate!
+
+ expect(migration).to have_received(:perform).with(1, 1).twice
+ end
+ end
+ end
+
+ context 'when there are no background migrations pending' do
+ it 'does nothing' do
+ Sidekiq::Testing.disable! do
+ migrate!
+
+ expect(migration).not_to have_received(:perform)
+ end
+ end
+ end
+
+ context 'when there are still unmigrated stages present' do
+ let(:stages) { table('ci_stages') }
+ let(:builds) { table('ci_builds') }
+
+ let!(:entities) do
+ %w[build test broken].map do |name|
+ stages.create(name: name)
+ end
+ end
+
+ before do
+ stages.update_all(position: nil)
+
+ builds.create(name: 'unit', stage_id: entities.first.id, stage_idx: 1, ref: 'master')
+ builds.create(name: 'unit', stage_id: entities.second.id, stage_idx: 1, ref: 'master')
+ end
+
+ it 'migrates stages sequentially for every stage' do
+ expect(stages.all).to all(have_attributes(position: nil))
+
+ migrate!
+
+ expect(migration).to have_received(:perform)
+ .with(entities.first.id, entities.first.id)
+ expect(migration).to have_received(:perform)
+ .with(entities.second.id, entities.second.id)
+ expect(migration).not_to have_received(:perform)
+ .with(entities.third.id, entities.third.id)
+ end
+ end
+end
diff --git a/vendor/project_templates/express.tar.gz b/vendor/project_templates/express.tar.gz
index 8dd5fa36987a8d349b4029c39d02b5c08dbafd81..fb357639a693472b0b24dd73bdcd5942041a3853 100644
Binary files a/vendor/project_templates/express.tar.gz and b/vendor/project_templates/express.tar.gz differ
diff --git a/vendor/project_templates/rails.tar.gz b/vendor/project_templates/rails.tar.gz
index 89337dc5c31abd82b8df58186ddcb17604f0bcf7..8454d2fc03bf56d40e83fb1e4826f69341662191 100644
Binary files a/vendor/project_templates/rails.tar.gz and b/vendor/project_templates/rails.tar.gz differ
diff --git a/vendor/project_templates/spring.tar.gz b/vendor/project_templates/spring.tar.gz
index 31c90d0820fbf95aaf84cfa1de8be7eec383a10c..55e25fdbe7cf3c7c7601213010e4c1a312ab9a8e 100644
Binary files a/vendor/project_templates/spring.tar.gz and b/vendor/project_templates/spring.tar.gz differ