From 35eecbf25f833538dd6e0abb816f1b1adcb7d19e Mon Sep 17 00:00:00 2001
From: Gabriel Mazetto <gabriel@gitlab.com>
Date: Fri, 17 Aug 2018 13:25:11 +0000
Subject: [PATCH] [EE] Fix SiteStatistics wikis_count
---
app/models/site_statistic.rb | 2 +-
.../unreleased/repopulate_site_statistics.yml | 5 +++
...9195358_migrate_null_wiki_access_levels.rb | 32 +++++++++++++++++++
db/schema.rb | 2 +-
lib/tasks/gitlab/site_statistics.rake | 23 +++++++++++++
.../bare_repository_import/importer_spec.rb | 4 ---
spec/lib/system_check/simple_executor_spec.rb | 9 ------
.../migrate_null_wiki_access_levels_spec.rb | 29 +++++++++++++++++
spec/spec_helper.rb | 1 +
.../tasks/gitlab/site_statistics_rake_spec.rb | 24 ++++++++++++++
10 files changed, 116 insertions(+), 15 deletions(-)
create mode 100644 changelogs/unreleased/repopulate_site_statistics.yml
create mode 100644 db/post_migrate/20180809195358_migrate_null_wiki_access_levels.rb
create mode 100644 lib/tasks/gitlab/site_statistics.rake
create mode 100644 spec/migrations/migrate_null_wiki_access_levels_spec.rb
create mode 100644 spec/tasks/gitlab/site_statistics_rake_spec.rb
diff --git a/app/models/site_statistic.rb b/app/models/site_statistic.rb
index daac1c57db907..48324570f0bd1 100644
--- a/app/models/site_statistic.rb
+++ b/app/models/site_statistic.rb
@@ -49,7 +49,7 @@ def self.with_statistics_available(raw_attribute)
#
# @return [SiteStatistic] record with tracked information
def self.fetch
- SiteStatistic.transaction(requires_new: true) do
+ transaction(requires_new: true) do
SiteStatistic.first_or_create!
end
rescue ActiveRecord::RecordNotUnique
diff --git a/changelogs/unreleased/repopulate_site_statistics.yml b/changelogs/unreleased/repopulate_site_statistics.yml
new file mode 100644
index 0000000000000..1961088061d8e
--- /dev/null
+++ b/changelogs/unreleased/repopulate_site_statistics.yml
@@ -0,0 +1,5 @@
+---
+title: Migrate NULL wiki_access_level to correct number so we count active wikis correctly
+merge_request: 21030
+author:
+type: changed
diff --git a/db/post_migrate/20180809195358_migrate_null_wiki_access_levels.rb b/db/post_migrate/20180809195358_migrate_null_wiki_access_levels.rb
new file mode 100644
index 0000000000000..0a0a33299e442
--- /dev/null
+++ b/db/post_migrate/20180809195358_migrate_null_wiki_access_levels.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+class MigrateNullWikiAccessLevels < ActiveRecord::Migration
+ include Gitlab::Database::MigrationHelpers
+
+ DOWNTIME = false
+
+ disable_ddl_transaction!
+
+ class ProjectFeature < ActiveRecord::Base
+ include EachBatch
+
+ self.table_name = 'project_features'
+ end
+
+ def up
+ ProjectFeature.where(wiki_access_level: nil).each_batch do |relation|
+ relation.update_all(wiki_access_level: 20)
+ end
+
+ # We need to re-count wikis as previous attempt was not considering the NULLs.
+ transaction do
+ execute('SET LOCAL statement_timeout TO 0') if Gitlab::Database.postgresql? # see https://gitlab.com/gitlab-org/gitlab-ce/issues/48967
+
+ execute("UPDATE site_statistics SET wikis_count = (SELECT COUNT(*) FROM project_features WHERE wiki_access_level != 0)")
+ end
+ end
+
+ def down
+ # there is no way to rollback this change, there are no downsides in keeping migrated data.
+ end
+end
diff --git a/db/schema.rb b/db/schema.rb
index edc1cfd1a0ffa..a74882e19e2dc 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -11,7 +11,7 @@
#
# It's strongly recommended that you check this file into your version control system.
-ActiveRecord::Schema.define(version: 20180808162000) do
+ActiveRecord::Schema.define(version: 20180809195358) do
# These are extensions that must be enabled in order to support this database
enable_extension "plpgsql"
diff --git a/lib/tasks/gitlab/site_statistics.rake b/lib/tasks/gitlab/site_statistics.rake
new file mode 100644
index 0000000000000..7d24ec72a9d41
--- /dev/null
+++ b/lib/tasks/gitlab/site_statistics.rake
@@ -0,0 +1,23 @@
+namespace :gitlab do
+ desc "GitLab | Refresh Site Statistics counters"
+ task refresh_site_statistics: :environment do
+ puts 'Updating Site Statistics counters: '
+
+ print '* Repositories... '
+ SiteStatistic.transaction do
+ # see https://gitlab.com/gitlab-org/gitlab-ce/issues/48967
+ ActiveRecord::Base.connection.execute('SET LOCAL statement_timeout TO 0') if Gitlab::Database.postgresql?
+ SiteStatistic.update_all('repositories_count = (SELECT COUNT(*) FROM projects)')
+ end
+ puts 'OK!'.color(:green)
+
+ print '* Wikis... '
+ SiteStatistic.transaction do
+ # see https://gitlab.com/gitlab-org/gitlab-ce/issues/48967
+ ActiveRecord::Base.connection.execute('SET LOCAL statement_timeout TO 0') if Gitlab::Database.postgresql?
+ SiteStatistic.update_all('wikis_count = (SELECT COUNT(*) FROM project_features WHERE wiki_access_level != 0)')
+ end
+ puts 'OK!'.color(:green)
+ puts
+ end
+end
diff --git a/spec/lib/gitlab/bare_repository_import/importer_spec.rb b/spec/lib/gitlab/bare_repository_import/importer_spec.rb
index 6e21c846c0ad6..3c63e601abc36 100644
--- a/spec/lib/gitlab/bare_repository_import/importer_spec.rb
+++ b/spec/lib/gitlab/bare_repository_import/importer_spec.rb
@@ -10,9 +10,6 @@
subject(:importer) { described_class.new(admin, bare_repository) }
before do
- @rainbow = Rainbow.enabled
- Rainbow.enabled = false
-
allow(described_class).to receive(:log)
end
@@ -20,7 +17,6 @@
FileUtils.rm_rf(base_dir)
TestEnv.clean_test_path
ensure_seeds
- Rainbow.enabled = @rainbow
end
shared_examples 'importing a repository' do
diff --git a/spec/lib/system_check/simple_executor_spec.rb b/spec/lib/system_check/simple_executor_spec.rb
index 9da3648400e00..e71e9da369d9d 100644
--- a/spec/lib/system_check/simple_executor_spec.rb
+++ b/spec/lib/system_check/simple_executor_spec.rb
@@ -98,15 +98,6 @@ def check?
end
end
- before do
- @rainbow = Rainbow.enabled
- Rainbow.enabled = false
- end
-
- after do
- Rainbow.enabled = @rainbow
- end
-
describe '#component' do
it 'returns stored component name' do
expect(subject.component).to eq('Test')
diff --git a/spec/migrations/migrate_null_wiki_access_levels_spec.rb b/spec/migrations/migrate_null_wiki_access_levels_spec.rb
new file mode 100644
index 0000000000000..f99273072a2f0
--- /dev/null
+++ b/spec/migrations/migrate_null_wiki_access_levels_spec.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20180809195358_migrate_null_wiki_access_levels.rb')
+
+describe MigrateNullWikiAccessLevels, :migration do
+ let(:namespaces) { table('namespaces') }
+ let(:projects) { table(:projects) }
+ let(:project_features) { table(:project_features) }
+ let(:migration) { described_class.new }
+
+ before do
+ namespace = namespaces.create(name: 'foo', path: 'foo')
+
+ projects.create!(id: 1, name: 'gitlab1', path: 'gitlab1', namespace_id: namespace.id)
+ projects.create!(id: 2, name: 'gitlab2', path: 'gitlab2', namespace_id: namespace.id)
+ projects.create!(id: 3, name: 'gitlab3', path: 'gitlab3', namespace_id: namespace.id)
+
+ project_features.create!(id: 1, project_id: 1, wiki_access_level: nil)
+ project_features.create!(id: 2, project_id: 2, wiki_access_level: 10)
+ project_features.create!(id: 3, project_id: 3, wiki_access_level: 20)
+ end
+
+ describe '#up' do
+ it 'migrates existing project_features with wiki_access_level NULL to 20' do
+ expect { migration.up }.to change { project_features.where(wiki_access_level: 20).count }.by(1)
+ end
+ end
+end
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index fb250526bcacb..198499237594f 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -29,6 +29,7 @@
# require rainbow gem String monkeypatch, so we can test SystemChecks
require 'rainbow/ext/string'
+Rainbow.enabled = false
require_relative '../ee/spec/spec_helper'
diff --git a/spec/tasks/gitlab/site_statistics_rake_spec.rb b/spec/tasks/gitlab/site_statistics_rake_spec.rb
new file mode 100644
index 0000000000000..20f0df65e63f6
--- /dev/null
+++ b/spec/tasks/gitlab/site_statistics_rake_spec.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+require 'rake_helper'
+
+describe 'rake gitlab:refresh_site_statistics' do
+ before do
+ Rake.application.rake_require 'tasks/gitlab/site_statistics'
+
+ create(:project)
+ SiteStatistic.fetch.update(repositories_count: 0, wikis_count: 0)
+ end
+
+ let(:task) { 'gitlab:refresh_site_statistics' }
+
+ it 'recalculates existing counters' do
+ run_rake_task(task)
+
+ expect(SiteStatistic.fetch.repositories_count).to eq(1)
+ expect(SiteStatistic.fetch.wikis_count).to eq(1)
+ end
+
+ it 'displays message listing counters' do
+ expect { run_rake_task(task) }.to output(/Updating Site Statistics counters:.* Repositories\.\.\. OK!.* Wikis\.\.\. OK!/m).to_stdout
+ end
+end
--
GitLab