Merge branch 'fetch_secrets_openbao_using_vault' into 'master'

Fetch secrets from OpenBao in CI pipeline

See merge request https://gitlab.com/gitlab-org/gitlab/-/merge_requests/162963



Merged-by: Erick Bajao <fbajao@gitlab.com>
Approved-by: Erick Bajao <fbajao@gitlab.com>
Reviewed-by: Fabio Pitino <fpitino@gitlab.com>
Reviewed-by: Erick Bajao <fbajao@gitlab.com>
Reviewed-by: Mireya Andres <mandres@gitlab.com>
Reviewed-by: Dmytro Biryukov <dbiryukov@gitlab.com>
Reviewed-by: Jonas Larsen <jlarsen@gitlab.com>
Reviewed-by: James Nutt <jnutt@gitlab.com>
Reviewed-by: Alex Scheel <ascheel@gitlab.com>
Co-authored-by: Alexander Scheel <ascheel@gitlab.com>
Co-authored-by: Shabini Rajadas <srajadas@gitlab.com>

app/validators/json_schemas/build_metadata_secrets.json

@@ -39,6 +39,18 @@
           },
           "additionalProperties": false
         },
+        "^gitlab_secrets_manager$": {
+          "type": "object",
+          "required": [
+            "name"
+          ],
+          "properties": {
+            "name": {
+              "type": "string"
+            }
+          },
+          "additionalProperties": false
+        },
         "^gcp_secret_manager$": {
           "type": "object",
           "required": [
@@ -198,6 +210,11 @@
           "required": [
             "akeyless"
           ]
+        },
+        {
+          "required": [
+            "gitlab_secrets_manager"
+          ]
         }
       ],
       "additionalProperties": false

ee/app/models/ci/secrets/integration.rb

@@ -3,19 +3,37 @@
 module Ci
   module Secrets
     class Integration
-      PROVIDERS = [
-        :azure_key_vault,
-        :akeyless,
-        :gcp_secret_manager,
-        :hashicorp_vault
-      ].freeze
-
-      def initialize(variables)
+      PROVIDER_TYPE_MAP = {
+        "azure_key_vault" => :azure_key_vault,
+        "akeyless" => :akeyless,
+        "gcp_secret_manager" => :gcp_secret_manager,
+        "vault" => :hashicorp_vault,
+        "gitlab_secrets_manager" => :gitlab_secrets_manager
+      }.freeze
+
+      def initialize(variables:, project:)
         @variables = variables
+        @project = project
       end
 
-      def secrets_provider?
-        PROVIDERS.any? { |provider| send(:"#{provider}?") } # rubocop:disable GitlabSecurity/PublicSend -- metaprogramming
+      def secrets_provider?(secrets)
+        candidates = PROVIDER_TYPE_MAP.values.select { |provider| send(:"#{provider}?") } # rubocop:disable GitlabSecurity/PublicSend -- metaprogramming
+
+        # No providers are enabled.
+        return false if candidates.empty?
+
+        # No secrets were provided; vacuously this means all provided secrets
+        # have a provider. This is deferred so global enablement logic can be
+        # checked independently of secrets value.
+        return true if secrets.nil? || secrets.empty?
+
+        # If none of the secrets lacks an enabled provider, we're good.
+        secrets.none? do |(_, secret_info)|
+          secret_info.any? do |(provider_key, _)|
+            PROVIDER_TYPE_MAP.has_key?(provider_key) &&
+              candidates.exclude?(PROVIDER_TYPE_MAP[provider_key])
+          end
+        end
       end
 
       def variable_value(key, default = nil)
@@ -51,6 +69,12 @@ def hashicorp_vault?
       def akeyless?
         variable_value('AKEYLESS_ACCESS_ID').present?
       end
+
+      def gitlab_secrets_manager?
+        # TODO: figure out context for whether GitLab Secrets Manager is
+        # globally enabled on this instance.
+        SecretsManagement::ProjectSecretsManager.find_by_project_id(@project.id)&.active?
+      end
     end
   end
 end

ee/app/models/ee/ci/build.rb

@@ -186,7 +186,7 @@ def runner_required_feature_names
       end
 
       def secrets_integration
-        ::Ci::Secrets::Integration.new(variables)
+        ::Ci::Secrets::Integration.new(variables: variables, project: project)
       end
 
       def playable?

ee/app/models/secrets_management/project_secrets_manager.rb

@@ -4,7 +4,8 @@ module SecretsManagement
   class ProjectSecretsManager < ApplicationRecord
     STATUSES = {
       provisioning: 0,
-      active: 1
+      active: 1,
+      disabled: 2
     }.freeze
 
     self.table_name = 'project_secrets_managers'
@@ -16,25 +17,76 @@ class ProjectSecretsManager < ApplicationRecord
     state_machine :status, initial: :provisioning do
       state :provisioning, value: STATUSES[:provisioning]
       state :active, value: STATUSES[:active]
+      state :disabled, value: STATUSES[:disabled]
 
       event :activate do
         transition all - [:active] => :active
       end
+
+      event :disable do
+        transition active: :disabled
+      end
+    end
+
+    def self.server_url
+      # Allow setting an external secrets manager URL if necessary. This is
+      # useful for GitLab.Com's deployment.
+      return Gitlab.config.openbao.url if Gitlab.config.has_key?("openbao") && Gitlab.config.openbao.has_key?("url")
+
+      default_openbao_server_url
+    end
+
+    def self.default_openbao_server_url
+      "#{Gitlab.config.gitlab.protocol}://#{Gitlab.config.gitlab.host}:8200"
     end
+    private_class_method :default_openbao_server_url
 
     def ci_secrets_mount_path
       [
         namespace_path,
         "project_#{project.id}",
-        'ci'
+        'secrets',
+        'kv'
       ].compact.join('/')
     end
 
+    def ci_data_path(secret_key)
+      [
+        'explicit',
+        secret_key
+      ].compact.join('/')
+    end
+
+    def ci_full_path(secret_key)
+      [
+        ci_secrets_mount_path,
+        'data',
+        ci_data_path(secret_key)
+      ].compact.join('/')
+    end
+
+    def ci_auth_mount
+      [
+        namespace_path,
+        'pipeline_jwt'
+      ].compact.join('/')
+    end
+
+    def ci_auth_role
+      "project_#{project.id}"
+    end
+
+    def ci_auth_type
+      'jwt'
+    end
+
+    def ci_jwt(build)
+      Gitlab::Ci::JwtV2.for_build(build, aud: self.class.server_url)
+    end
+
     private
 
     def namespace_path
-      return unless project.namespace.type == "User"
-
       [
         project.namespace.type.downcase,
         project.namespace.id.to_s

ee/app/presenters/ee/ci/build_runner_presenter.rb

@@ -15,6 +15,26 @@ def secrets_configuration
             secret['akeyless']['server'] = akeyless_server(secret)
           end
 
+          # For compatibility with the existing Vault integration in Runner,
+          # template gitlab_secrets_manager data into the vault field.
+          if secret.has_key?('gitlab_secrets_manager')
+            # GitLab Secrets Manager and Vault integrations have different
+            # structure; remove the old secret but save its data for later.
+            gtsm_secret = secret.delete('gitlab_secrets_manager')
+
+            psm = SecretsManagement::ProjectSecretsManager.find_by_project_id(project.id)
+
+            # Compute full path to secret in OpenBao for Vault runner
+            # compatibility.
+            secret['vault'] = {}
+            secret['vault']['path'] = psm.ci_data_path(gtsm_secret['name'])
+            secret['vault']['engine'] = { name: "kv-v2", path: psm.ci_secrets_mount_path }
+            secret['vault']['field'] = "value"
+
+            # Tell Runner about our server information.
+            secret['vault']['server'] = gitlab_secrets_manager_server(psm)
+          end
+
           secret
         end
       end
@@ -36,6 +56,20 @@ def vault_server(secret)
         }
       end
 
+      def gitlab_secrets_manager_server(psm)
+        @gitlab_secrets_manager_server ||= {
+          'url' => SecretsManagement::ProjectSecretsManager.server_url,
+          'auth' => {
+            'name' => psm.ci_auth_type,
+            'path' => psm.ci_auth_mount,
+            'data' => {
+              'jwt' => psm.ci_jwt(self),
+              'role' => psm.ci_auth_role
+            }.compact
+          }
+        }
+      end
+
       def vault_jwt(secret)
         if id_tokens?
           id_token_var(secret)

ee/app/services/ci/pipeline_creation/drop_secrets_provider_not_found_builds_service.rb

@@ -12,7 +12,7 @@ def execute
 
         pipeline.builds.each do |build|
           next unless build.created?
-          next unless build.secrets? && !build.secrets_provider?
+          next unless build.secrets? && !build.secrets_provider?(build.secrets)
 
           build.drop!(:secrets_provider_not_found, skip_pipeline_processing: true)
         end

ee/config/events/create_secrets_gitlab_secrets_manager.yml

+---
+description: Pipeline created with secrets (GitLab Secrets Manager) defined
+internal_events: true
+action: create_secrets_gitlab_secrets_manager
+identifiers:
+- namespace
+- user
+product_group: pipeline_security
+milestone: '17.6'
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/162963
+distributions:
+- ee
+tiers:
+- premium
+- ultimate

ee/config/metrics/counts_28d/20210216184251_i_ci_secrets_management_vault_build_created_monthly.yml

@@ -2,7 +2,7 @@
 data_category: optional
 key_path: redis_hll_counters.ci_secrets_management.i_ci_secrets_management_vault_build_created_monthly
 description: Monthly active users creating pipelines that that have the Vault JWT
-  with it.'
+  with it.
 product_group: environments
 value_type: number
 status: active

ee/config/metrics/counts_28d/count_total_create_secrets_azure_key_vault_monthly.yml

 ---
 data_category: optional
 key_path: redis_hll_counters.ci_secrets_management.i_ci_secrets_management_azure_key_vault_build_created_monthly
-description: Monthly active users creating pipelines that that have the Azure Key Vault secrets.'
+description: Monthly active users creating pipelines that that have the Azure Key Vault secrets.
 product_group: pipeline_security
 value_type: number
 status: active

ee/config/metrics/counts_28d/count_total_create_secrets_gitlab_secrets_manager_monthly.yml

+---
+data_category: optional
+key_path: redis_hll_counters.ci_secrets_management.i_ci_secrets_management_gitlab_secrets_manager_build_created_monthly
+description: Monthly active users creating pipelines that that have the GitLab Secret Manager secrets.
+product_group: pipeline_security
+value_type: number
+status: active
+time_frame: 28d
+data_source: redis_hll
+instrumentation_class: RedisHLLMetric
+options:
+  events:
+    - i_ci_secrets_management_gitlab_secrets_manager_build_created
+distribution:
+  - ee
+tier:
+  - premium
+  - ultimate
+performance_indicator_type: []
+milestone: "17.6"
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/162963

ee/config/metrics/counts_28d/count_total_create_secrets_google_cloud_monthly.yml

 ---
 data_category: optional
 key_path: redis_hll_counters.ci_secrets_management.i_ci_secrets_management_gcp_secret_manager_build_created_monthly
-description: Monthly active users creating pipelines that that have the GCP Secret Manager secrets.'
+description: Monthly active users creating pipelines that that have the GCP Secret Manager secrets.
 product_group: pipeline_security
 value_type: number
 status: active

ee/config/metrics/counts_7d/20210216184249_i_ci_secrets_management_vault_build_created_weekly.yml

@@ -2,7 +2,7 @@
 data_category: optional
 key_path: redis_hll_counters.ci_secrets_management.i_ci_secrets_management_vault_build_created_weekly
 description: Weekly active users creating pipelines that that have the Vault JWT
-  with it.'
+  with it.
 product_group: environments
 value_type: number
 status: active

ee/config/metrics/counts_7d/count_total_create_secrets_azure_key_vault_weekly.yml

 ---
 data_category: optional
 key_path: redis_hll_counters.ci_secrets_management.i_ci_secrets_management_azure_key_vault_build_created_weekly
-description: Monthly active users creating pipelines that that have the Azure Key Vault secrets.'
+description: Monthly active users creating pipelines that that have the Azure Key Vault secrets.
 product_group: pipeline_security
 value_type: number
 status: active

ee/config/metrics/counts_7d/count_total_create_secrets_gitlab_secrets_manager_weekly.yml

+---
+data_category: optional
+key_path: redis_hll_counters.ci_secrets_management.i_ci_secrets_management_gitlab_secrets_manager_build_created_weekly
+description: Monthly active users creating pipelines that that have the GitLab Secrets Manager secrets.
+product_group: pipeline_security
+value_type: number
+status: active
+time_frame: 7d
+data_source: redis_hll
+instrumentation_class: RedisHLLMetric
+options:
+  events:
+    - i_ci_secrets_management_gitlab_secrets_manager_build_created
+distribution:
+  - ee
+tier:
+  - premium
+  - ultimate
+performance_indicator_type: []
+milestone: "17.6"
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/162963

ee/config/metrics/counts_7d/count_total_create_secrets_google_cloud_weekly.yml

 ---
 data_category: optional
 key_path: redis_hll_counters.ci_secrets_management.i_ci_secrets_management_gcp_secret_manager_build_created_weekly
-description: Monthly active users creating pipelines that that have the GCP Secret Manager secrets.'
+description: Monthly active users creating pipelines that that have the GCP Secret Manager secrets.
 product_group: pipeline_security
 value_type: number
 status: active

ee/lib/gitlab/ci/config/entry/gitlab_secrets_manager/secret.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        module GitlabSecretsManager
+          class Secret < ::Gitlab::Config::Entry::Node
+            include ::Gitlab::Config::Entry::Validatable
+            include ::Gitlab::Config::Entry::Attributable
+
+            ALLOWED_KEYS = %i[name].freeze
+
+            attributes ALLOWED_KEYS
+
+            validations do
+              validates :config, type: Hash
+              validates :name, presence: true, type: String
+            end
+
+            def value
+              {
+                name: name
+              }
+            end
+          end
+        end
+      end
+    end
+  end
+end

ee/lib/gitlab/ci/config/entry/secret.rb

@@ -11,8 +11,8 @@ class Secret < ::Gitlab::Config::Entry::Node
           include ::Gitlab::Config::Entry::Configurable
           include ::Gitlab::Config::Entry::Attributable
 
-          ALLOWED_KEYS = %i[vault file azure_key_vault gcp_secret_manager akeyless token].freeze
-          SUPPORTED_PROVIDERS = %i[vault azure_key_vault gcp_secret_manager akeyless].freeze
+          ALLOWED_KEYS = %i[vault file azure_key_vault gcp_secret_manager akeyless gitlab_secrets_manager token].freeze
+          SUPPORTED_PROVIDERS = %i[vault azure_key_vault gcp_secret_manager akeyless gitlab_secrets_manager].freeze
 
           attributes ALLOWED_KEYS
 
@@ -21,6 +21,8 @@ class Secret < ::Gitlab::Config::Entry::Node
           entry :azure_key_vault, Entry::AzureKeyVault::Secret, description: 'Azure Key Vault configuration'
           entry :gcp_secret_manager, Entry::GcpSecretManager::Secret, description: 'GCP Secrets Manager configuration'
           entry :akeyless, Entry::Akeyless::Secret, description: 'Akeyless Key Vault configuration'
+          entry :gitlab_secrets_manager, Entry::GitlabSecretsManager::Secret,
+            description: 'Gitlab Secrets Manager configuration'
 
           validations do
             validates :config, allowed_keys: ALLOWED_KEYS, only_one_of_keys: SUPPORTED_PROVIDERS
@@ -34,6 +36,7 @@ class Secret < ::Gitlab::Config::Entry::Node
           def value
             {
               vault: vault_value,
+              gitlab_secrets_manager: gitlab_secrets_manager_value,
               gcp_secret_manager: gcp_secret_manager_value,
               azure_key_vault: azure_key_vault_value,
               akeyless: akeyless_value,

ee/spec/lib/gitlab/ci/config/entry/gitlab_secrets_manager/secret_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Ci::Config::Entry::GitlabSecretsManager::Secret, feature_category: :secrets_management do
+  let(:entry) { described_class.new(config) }
+
+  before do
+    entry.compose!
+  end
+
+  describe 'validations' do
+    context 'when all config value is correct' do
+      let(:config) do
+        {
+          name: 'name'
+        }
+      end
+
+      it { expect(entry).to be_valid }
+    end
+
+    context 'when name is nil' do
+      let(:config) do
+        {
+          name: nil
+        }
+      end
+
+      it { expect(entry).not_to be_valid }
+
+      it 'reports error' do
+        expect(entry.errors)
+          .to include 'secret name can\'t be blank'
+      end
+    end
+
+    context 'when there is an unknown key present' do
+      let(:config) { { foo: :bar } }
+
+      it { expect(entry).not_to be_valid }
+
+      it 'reports error' do
+        expect(entry.errors)
+          .to include "secret name can't be blank"
+      end
+    end
+
+    context 'when config is not a hash' do
+      let(:config) { "" }
+
+      it { expect(entry).not_to be_valid }
+
+      it 'reports error' do
+        expect(entry.errors)
+          .to include 'secret config should be a hash'
+      end
+    end
+  end
+
+  describe '#value' do
+    context 'when config is valid' do
+      let(:config) do
+        {
+          name: 'name'
+        }
+      end
+
+      let(:result) do
+        {
+          name: "name"
+        }
+      end
+
+      it 'returns config' do
+        expect(entry.value).to eq(result)
+      end
+    end
+  end
+end

ee/spec/lib/gitlab/ci/config/entry/secret_spec.rb

@@ -299,6 +299,36 @@
           end
         end
       end
+
+      context 'for Gitlab Secrets Manager' do
+        context 'when config is valid' do
+          let(:config) do
+            {
+              gitlab_secrets_manager: {
+                name: 'name'
+              }
+            }
+          end
+
+          describe '#value' do
+            it 'returns secret configuration' do
+              expected_result = {
+                gitlab_secrets_manager: {
+                  name: 'name'
+                }
+              }
+
+              expect(entry.value).to eq(expected_result)
+            end
+          end
+
+          describe '#valid?' do
+            it 'is valid' do
+              expect(entry).to be_valid
+            end
+          end
+        end
+      end
     end
   end
 
@@ -319,7 +349,7 @@
         it 'reports error' do
           expect(entry.errors)
             .to include 'secret config must use exactly one of these keys: ' \
-            'vault, azure_key_vault, gcp_secret_manager, akeyless'
+            'vault, azure_key_vault, gcp_secret_manager, akeyless, gitlab_secrets_manager'
         end
       end
 

ee/spec/lib/gitlab/ci/yaml_processor_spec.rb

@@ -304,6 +304,32 @@
       end
     end
 
+    context 'on gitlab_secrets_manager' do
+      let(:secrets) do
+        {
+          DATABASE_PASSWORD: {
+            gitlab_secrets_manager: {
+              name: 'password'
+            }
+          }
+        }
+      end
+
+      let(:config) { { deploy_to_production: { stage: 'deploy', script: ['echo'], secrets: secrets } } }
+
+      it "returns secrets info" do
+        secrets = result.builds.first.fetch(:secrets)
+
+        expect(secrets).to eq({
+          DATABASE_PASSWORD: {
+            gitlab_secrets_manager: {
+              name: 'password'
+            }
+          }
+        })
+      end
+    end
+
     context 'on azure key vault' do
       let(:secrets) do
         {