Merge branch '356986-aw-add-token-sub-keyword' into 'master'

Add `token` sub-keyword to CI config

See merge request https://gitlab.com/gitlab-org/gitlab/-/merge_requests/103801



Merged-by: Marius Bobin <mbobin@gitlab.com>
Approved-by: Mireya Andres <mandres@gitlab.com>
Approved-by: Marcel Amirault <mamirault@gitlab.com>
Approved-by: Marius Bobin <mbobin@gitlab.com>
Reviewed-by: Furkan Ayhan <furkanayhn@gmail.com>
Reviewed-by: Marius Bobin <mbobin@gitlab.com>
Reviewed-by: Avielle Wolfe <awolfe@gitlab.com>
Reviewed-by: Marcel Amirault <mamirault@gitlab.com>
Reviewed-by: Mireya Andres <mandres@gitlab.com>
Co-authored-by: Furkan Ayhan <furkanayhn@gmail.com>
Co-authored-by: Avielle Wolfe <awolfe@gitlab.com>
Co-authored-by: Mireya Andres <mandres@gitlab.com>

app/assets/javascripts/editor/schema/ci.json

@@ -607,53 +607,65 @@
     "secrets": {
       "type": "object",
       "markdownDescription": "Defines secrets to be injected as environment variables. [Learn More](https://docs.gitlab.com/ee/ci/yaml/#secrets).",
-      "additionalProperties": {
-        "type": "object",
-        "description": "Environment variable name",
-        "properties": {
-          "vault": {
-            "oneOf": [
-              {
-                "type": "string",
-                "description": "The secret to be fetched from Vault (e.g. 'production/db/password@ops' translates to secret 'ops/data/production/db', field `password`)"
-              },
-              {
-                "type": "object",
-                "properties": {
-                  "engine": {
-                    "type": "object",
-                    "properties": {
-                      "name": {
-                        "type": "string"
+      "patternProperties": {
+        ".*": {
+          "type": "object",
+          "properties": {
+            "vault": {
+              "oneOf": [
+                {
+                  "type": "string",
+                  "markdownDescription": "The secret to be fetched from Vault (e.g. 'production/db/password@ops' translates to secret 'ops/data/production/db', field `password`). [Learn More](https://docs.gitlab.com/ee/ci/yaml/#secretsvault)"
+                },
+                {
+                  "type": "object",
+                  "properties": {
+                    "engine": {
+                      "type": "object",
+                      "properties": {
+                        "name": {
+                          "type": "string"
+                        },
+                        "path": {
+                          "type": "string"
+                        }
                       },
-                      "path": {
-                        "type": "string"
-                      }
+                      "required": [
+                        "name",
+                        "path"
+                      ]
                     },
-                    "required": [
-                      "name",
-                      "path"
-                    ]
-                  },
-                  "path": {
-                    "type": "string"
+                    "path": {
+                      "type": "string"
+                    },
+                    "field": {
+                      "type": "string"
+                    }
                   },
-                  "field": {
-                    "type": "string"
-                  }
-                },
-                "required": [
-                  "engine",
-                  "path",
-                  "field"
-                ]
-              }
-            ]
-          }
-        },
-        "required": [
-          "vault"
-        ]
+                  "required": [
+                    "engine",
+                    "path",
+                    "field"
+                  ],
+                  "additionalProperties": false
+                }
+              ]
+            },
+            "file": {
+              "type": "boolean",
+              "default": true,
+              "markdownDescription": "Configures the secret to be stored as either a file or variable type CI/CD variable. [Learn More](https://docs.gitlab.com/ee/ci/yaml/#secretsfile)"
+            },
+            "token": {
+              "type": "string",
+              "description": "Specifies the JWT variable that should be used to authenticate with Hashicorp Vault."
+            }
+          },
+          "required": [
+            "vault"
+          ],
+          "additionalProperties": false
+        }
       }
     },
     "before_script": {

ee/app/presenters/ee/ci/build_runner_presenter.rb

@@ -40,9 +40,7 @@ def vault_jwt(secret)
       def id_token_var(secret)
         return unless id_tokens?
 
-        token = secret['token'] || id_tokens.each_key.first
-
-        "${#{token}}"
+        secret['token'] || "$#{id_tokens.each_key.first}"
       end
     end
   end

ee/lib/gitlab/ci/config/entry/secret.rb

@@ -12,7 +12,7 @@ class Secret < ::Gitlab::Config::Entry::Node
           include ::Gitlab::Config::Entry::Attributable
 
           REQUIRED_KEYS = %i[vault].freeze
-          ALLOWED_KEYS = (REQUIRED_KEYS + %i[file]).freeze
+          ALLOWED_KEYS = (REQUIRED_KEYS + %i[file token]).freeze
 
           attributes ALLOWED_KEYS
 
@@ -21,6 +21,15 @@ class Secret < ::Gitlab::Config::Entry::Node
 
           validations do
             validates :config, allowed_keys: ALLOWED_KEYS, required_keys: REQUIRED_KEYS
+            validates :token, type: String, allow_nil: true
+          end
+
+          def value
+            {
+              vault: vault_value,
+              file: file_value,
+              token: token
+            }.compact
           end
         end
       end

ee/spec/lib/gitlab/ci/config/entry/secret_spec.rb

@@ -53,6 +53,40 @@
 
         it_behaves_like 'configures secrets'
       end
+
+      context 'when `token` is defined' do
+        let(:config) do
+          {
+            vault: {
+              engine: { name: 'kv-v2', path: 'kv-v2' },
+              path: 'production/db',
+              field: 'password'
+            },
+            token: '$TEST_ID_TOKEN'
+          }
+        end
+
+        describe '#value' do
+          it 'returns secret configuration' do
+            expect(entry.value).to eq(
+              {
+                vault: {
+                  engine: { name: 'kv-v2', path: 'kv-v2' },
+                  path: 'production/db',
+                  field: 'password'
+                },
+                token: '$TEST_ID_TOKEN'
+              }
+            )
+          end
+        end
+
+        describe '#valid?' do
+          it 'is valid' do
+            expect(entry).to be_valid
+          end
+        end
+      end
     end
   end
 

ee/spec/presenters/ci/build_runner_presenter_spec.rb

@@ -127,7 +127,7 @@
           it 'adds the first ID token to the Vault server payload' do
             jwt = presenter.secrets_configuration.dig('DATABASE_PASSWORD', 'vault', 'server', 'auth', 'data', 'jwt')
 
-            expect(jwt).to eq('${VAULT_ID_TOKEN_1}')
+            expect(jwt).to eq('$VAULT_ID_TOKEN_1')
           end
 
           context 'when the token variable is specified for the vault secret' do
@@ -135,7 +135,7 @@
               {
                 DATABASE_PASSWORD: {
                   file: true,
-                  token: 'VAULT_ID_TOKEN_2',
+                  token: '$VAULT_ID_TOKEN_2',
                   vault: {
                     engine: { name: 'kv-v2', path: 'kv-v2' },
                     path: 'production/db',
@@ -148,7 +148,7 @@
             it 'uses the specified token variable' do
               jwt = presenter.secrets_configuration.dig('DATABASE_PASSWORD', 'vault', 'server', 'auth', 'data', 'jwt')
 
-              expect(jwt).to eq('${VAULT_ID_TOKEN_2}')
+              expect(jwt).to eq('$VAULT_ID_TOKEN_2')
             end
           end
         end

ee/spec/services/ci/create_pipeline_service_spec.rb

@@ -155,6 +155,7 @@
           secrets:
             DATABASE_PASSWORD:
               vault: production/db/password
+              token: $ID_TOKEN
       YAML
     end
 
@@ -172,7 +173,8 @@
             'engine' => { 'name' => 'kv-v2', 'path' => 'kv-v2' },
             'path' => 'production/db',
             'field' => 'password'
-          }
+          },
+          'token' => '$ID_TOKEN'
         }
       })
     end

spec/frontend/editor/schema/ci/ci_schema_spec.js

@@ -32,6 +32,7 @@ import VariablesYaml from './yaml_tests/positive_tests/variables.yml';
 import JobWhenYaml from './yaml_tests/positive_tests/job_when.yml';
 import IdTokensYaml from './yaml_tests/positive_tests/id_tokens.yml';
 import HooksYaml from './yaml_tests/positive_tests/hooks.yml';
+import SecretsYaml from './yaml_tests/positive_tests/secrets.yml';
 
 // YAML NEGATIVE TEST
 import ArtifactsNegativeYaml from './yaml_tests/negative_tests/artifacts.yml';
@@ -50,6 +51,7 @@ import VariablesInvalidSyntaxDescYaml from './yaml_tests/negative_tests/variable
 import VariablesWrongSyntaxUsageExpand from './yaml_tests/negative_tests/variables/wrong_syntax_usage_expand.yml';
 import IdTokensNegativeYaml from './yaml_tests/negative_tests/id_tokens.yml';
 import HooksNegative from './yaml_tests/negative_tests/hooks.yml';
+import SecretsNegativeYaml from './yaml_tests/negative_tests/secrets.yml';
 
 const ajv = new Ajv({
   strictTypes: false,
@@ -87,6 +89,7 @@ describe('positive tests', () => {
       VariablesYaml,
       ProjectPathYaml,
       IdTokensYaml,
+      SecretsYaml,
     }),
   )('schema validates %s', (_, input) => {
     // We construct a new "JSON" from each main key that is inside a
@@ -122,6 +125,7 @@ describe('negative tests', () => {
       ProjectPathIncludeLeadSlashYaml,
       ProjectPathIncludeNoSlashYaml,
       ProjectPathIncludeTailSlashYaml,
+      SecretsNegativeYaml,
       TriggerNegative,
       HooksNegative,
     }),

spec/frontend/editor/schema/ci/yaml_tests/negative_tests/secrets.yml

+job_with_secrets_without_vault:
+  script:
+    - echo $TEST_DB_PASSWORD
+  secrets:
+    TEST_DB_PASSWORD:
+      token: $TEST_TOKEN
+
+job_with_secrets_with_extra_properties:
+  script:
+    - echo $TEST_DB_PASSWORD
+  secrets:
+    TEST_DB_PASSWORD:
+      vault: test/db/password
+      extra_prop: TEST
+
+job_with_secrets_with_invalid_vault_property:
+  script:
+    - echo $TEST_DB_PASSWORD
+  secrets:
+    TEST_DB_PASSWORD:
+      vault:
+        invalid: TEST
+
+job_with_secrets_with_missing_required_vault_property:
+  script:
+    - echo $TEST_DB_PASSWORD
+  secrets:
+    TEST_DB_PASSWORD:
+      vault:
+        path: gitlab
+
+job_with_secrets_with_missing_required_engine_property:
+  script:
+    - echo $TEST_DB_PASSWORD
+  secrets:
+    TEST_DB_PASSWORD:
+      vault:
+        engine:
+          path: kv

spec/frontend/editor/schema/ci/yaml_tests/positive_tests/secrets.yml

+valid_job_with_secrets:
+  script:
+    - echo $TEST_DB_PASSWORD
+  secrets:
+    TEST_DB_PASSWORD:
+      vault: test/db/password
+
+valid_job_with_secrets_and_token:
+  script:
+    - echo $TEST_DB_PASSWORD
+  secrets:
+    TEST_DB_PASSWORD:
+      vault: test/db/password
+      token: $TEST_TOKEN
+
+valid_job_with_secrets_with_every_vault_keyword:
+  script:
+    - echo $TEST_DB_PASSWORD
+  secrets:
+    TEST_DB_PASSWORD:
+      vault:
+        engine:
+          name: test-engine
+          path: test
+        path: test/db
+        field: password
+      file: true
+      token: $TEST_TOKEN