Merge branch 'cherry-pick-734e8efc-to-16-8' into '16-8-stable-ee'

Cherry pick add job to re-tag gdk image on release tag creation to 16.8

See merge request https://gitlab.com/gitlab-org/gitlab/-/merge_requests/153490



Merged-by: Stan Hu <stanhu@gmail.com>
Co-authored-by: Alan (Maciej) Paruszewski <mparuszewski@gitlab.com>
Co-authored-by: Jennifer Li <jli@gitlab.com>
Co-authored-by: GitLab Renovate Bot <gitlab-bot@gitlab.com>
Co-authored-by: Andrejs Cunskis <acunskis@gitlab.com>
Co-authored-by: GitLab Dependency Bot <leipert+gitlab-renovate-bot@gitlab.com>

.gitlab/ci/build-images.gitlab-ci.yml

@@ -29,6 +29,21 @@ build-qa-image as-if-foss:
     - .as-if-foss
     - .build-images:rules:build-qa-image-as-if-foss
 
+retag-gdk-image:
+  extends:
+    - .base-image-build
+    - .build-images:rules:retag-gdk-image
+  tags:
+    - docker
+  stage: build-images
+  needs: []
+  script:
+    - |
+      image="registry.gitlab.com/gitlab-org/gitlab-development-kit/asdf-bootstrapped-verify/main"
+      tag=$(awk '/ARG GDK_SHA=/ {print $2}' qa/gdk/Dockerfile.gdk | sed 's/.*=//g')
+      skopeo login -u $RETAG_GDK_IMAGE_TOKEN_NAME -p $RETAG_GDK_IMAGE_TOKEN $CI_REGISTRY
+      skopeo copy docker://${image}:${tag} docker://${image}:stable-${tag}
+
 build-gdk-image:
   extends:
     - .base-image-build-buildx
@@ -36,7 +51,9 @@ build-gdk-image:
   tags:
     - saas-linux-xlarge-amd64
   stage: build-images
-  needs: []
+  needs:
+    - job: retag-gdk-image
+      optional: true
   script:
     - run_timed_command "scripts/build_gdk_image"
 

.gitlab/ci/rules.gitlab-ci.yml

@@ -923,6 +923,16 @@
       variables:
         ARCH: amd64,arm64
 
+# When new minor release tag is pushed, re-tag gdk image with pattern ignored by registry cleanup
+.build-images:rules:retag-gdk-image:
+  rules:
+    - !reference [".qa:rules:package-and-test-never-run", rules]
+    - if: '$CI_COMMIT_TAG =~ /^v\d+\.\d+\.0-ee$/ && $CI_PIPELINE_SOURCE == "push"'
+    # In case gdk base tag is updated via backport mr, make sure we retag it with stable prefix as well
+    - if: '$CI_MERGE_REQUEST_IID && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME =~ /^[\d-]+-stable-ee$/'
+      changes:
+        - qa/gdk/Dockerfile.gdk
+
 # We use a multi-stage image to:
 # - (re)build the first stage in master pipelines (including scheduled pipelines), and
 # - build the final stage in code-change pipelines (including MRs), and scheduled pipelines

ee/app/services/security/security_orchestration_policies/project_create_service.rb

@@ -34,13 +34,27 @@ def unlinked_project_exists?
       end
 
       def add_members(policy_project)
-        members_to_add = developers_and_maintainers - policy_project.team.members
-        policy_project.add_members(members_to_add, :developer) || []
+        members_to_add = developers_and_maintainers_without_group_access(policy_project)
+        policy_project.add_members(members_to_add, :developer)
       end
 
-      def developers_and_maintainers
-        container.team.members_with_access_levels(ACCESS_LEVELS_TO_ADD)
+      # rubocop:disable CodeReuse/ActiveRecord -- too specific for a scope
+      def developers_and_maintainers_without_group_access(policy_project)
+        user_ids = ProjectAuthorization
+                     .where(project_id: container.id, access_level: ACCESS_LEVELS_TO_ADD)
+                     .where_not_exists(
+                       GroupMember
+                         .where("members.user_id = project_authorizations.user_id")
+                         .where(source_id: policy_project.namespace.self_and_ancestor_ids,
+                           access_level: ::Gitlab::Access::DEVELOPER...))
+                     .distinct
+                     .pluck(:user_id) # rubocop:disable Database/AvoidUsingPluckWithoutLimit -- avoids cross-join
+
+        return User.none if user_ids.none?
+
+        User.where(id: user_ids)
       end
+      # rubocop:enable CodeReuse/ActiveRecord
 
       def create_project_params
         {

ee/spec/fixtures/projects/security/policies/expected_readme_for_project.md

@@ -36,6 +36,6 @@ You can read more about the format and policies schema in the [documentation](ht
 ## Default branch protection settings
 
 This project is preconfigured with the default branch set as a protected branch, and only maintainers/owners of
-[Target Project](http://localhost/target-namespace/target-project) have permission to merge into that branch. This overrides any default branch protection both at the
+[Target Project](http://localhost/target-group/target-project) have permission to merge into that branch. This overrides any default branch protection both at the
 [group level](http://localhost/help/user/group/manage#change-the-default-branch-protection-of-a-group) and at the
 [instance level](http://localhost/help/user/project/repository/branches/default#instance-level-default-branch-protection).

ee/spec/services/security/security_orchestration_policies/project_create_service_spec.rb

@@ -4,13 +4,12 @@
 
 RSpec.describe Security::SecurityOrchestrationPolicies::ProjectCreateService, feature_category: :security_policy_management do
   describe '#execute' do
-    let_it_be(:namespace) { create(:namespace, path: 'target-namespace') }
+    let_it_be(:group) { create(:group, path: 'target-group', name: 'Target Group') }
 
     let_it_be_with_refind(:project) do
-      create(:project, path: 'target-project', name: 'Target Project', namespace: namespace)
+      create(:project, path: 'target-project', name: 'Target Project', group: group)
     end
 
-    let_it_be(:group) { create(:group, path: 'target-group', name: 'Target Group') }
     let_it_be(:owner) { create(:user) }
     let_it_be(:maintainer) { create(:user) }
     let_it_be(:developer) { create(:user) }
@@ -20,13 +19,15 @@
       File.read(Rails.root.join(path))
     end
 
-    let(:current_user) { container.first_owner }
+    let(:current_user) { owner }
     let(:container) { project }
 
     subject(:service) { described_class.new(container: container, current_user: current_user) }
 
     before do
       stub_licensed_features(security_orchestration_policies: true)
+      group.add_owner(owner)
+      container.add_owner(owner)
     end
 
     context 'when security_orchestration_policies_configuration does not exist for project' do
@@ -64,7 +65,6 @@
       let(:container) { group }
 
       before_all do
-        group.add_owner(owner)
         group.add_maintainer(maintainer)
         group.add_developer(developer)
       end
@@ -82,9 +82,22 @@
       end
     end
 
-    context 'when adding users to security policy project fails' do
-      let(:current_user) { project.first_owner }
+    context 'when user is added as maintainer to both group and the project' do
+      let(:current_user) { owner }
+
+      before_all do
+        group.add_maintainer(maintainer)
+        project.add_maintainer(maintainer)
+      end
+
+      it 'successfully create projects without errors' do
+        response = service.execute
 
+        expect(response[:status]).to eq(:success)
+      end
+    end
+
+    context 'when adding users to security policy project fails' do
       before_all do
         project.add_maintainer(maintainer)
       end
@@ -144,7 +157,6 @@
       let(:container) { group }
 
       before do
-        group.add_owner(owner)
         group.update_attribute(:project_creation_level, Gitlab::Access::NO_ACCESS)
       end
 

qa/gdk/Dockerfile.gdk

-FROM registry.gitlab.com/gitlab-org/gitlab-development-kit/asdf-bootstrapped-verify/main:cb5028c5070c4ba17985d26326e941a058442304 as base
+ARG GDK_SHA=e8a2f3c30d3031c5574957efbaa917bacdeb3f1a
+# Use tag prefix when running on 'stable' branch to make sure 'protected' image is used which is not deleted by registry cleanup
+ARG GDK_BASE_TAG_PREFIX
+
+FROM registry.gitlab.com/gitlab-org/gitlab-development-kit/asdf-bootstrapped-verify/main:${GDK_BASE_TAG_PREFIX}${GDK_SHA} as base
 
 ENV GITLAB_LICENSE_MODE=test \
     GDK_KILL_CONFIRM=true
@@ -53,7 +57,7 @@ RUN set -eux; \
 #
 FROM base as gitlab-gems
 
-COPY --chown=gdk:gdk Gemfile Gemfile.lock ./gitlab/
+COPY --chown=gdk:gdk Gemfile Gemfile.lock .tool-versions ./gitlab/
 COPY --chown=gdk:gdk vendor/gems/ ./gitlab/vendor/gems/
 COPY --chown=gdk:gdk gems/ ./gitlab/gems/
 RUN make .gitlab-bundle \

scripts/build_gdk_image

@@ -7,6 +7,7 @@ source "$(dirname "$0")/utils.sh"
 REGISTRY="${CI_REGISTRY}/${CI_PROJECT_PATH}"
 SHA_TAG="${CI_COMMIT_SHA}"
 BRANCH_TAG="${CI_COMMIT_REF_SLUG}"
+STABLE_BRANCH_PATTERN="^[\d-]+-stable-ee$"
 
 IMAGE="${REGISTRY}/gitlab-qa-gdk"
 
@@ -16,6 +17,10 @@ else
   OUTPUT_OPTION="--load"
 fi
 
+if [[ "$BRANCH_TAG" =~ $STABLE_BRANCH_PATTERN || "$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" =~ $STABLE_BRANCH_PATTERN ]]; then
+  GDK_BASE_TAG_PREFIX="stable-"
+fi
+
 echoinfo "Building GDK image" "yes"
 
 docker buildx build \
@@ -24,6 +29,7 @@ docker buildx build \
   --cache-from="${IMAGE}/cache:master" \
   --file="qa/gdk/Dockerfile.gdk" \
   --platform=${ARCH:-amd64} \
+  --build-arg "GDK_BASE_TAG_PREFIX=${GDK_BASE_TAG_PREFIX}" \
   --tag="${IMAGE}:${SHA_TAG}" \
   --tag="${IMAGE}:${BRANCH_TAG}" \
   --provenance=false \

spec/lib/gitlab/metrics/exporter/base_exporter_spec.rb

@@ -216,7 +216,7 @@ def call(env)
       # in separate thread
       allow_any_instance_of(::WEBrick::HTTPServer)
         .to receive(:start).and_wrap_original do |m, *args|
-        Thread.new do
+        @request_thread = Thread.new do # rubocop:disable RSpec/InstanceVariable -- allowing for now
           m.call(*args)
         rescue IOError
           # is raised as we close listeners
@@ -226,6 +226,7 @@ def call(env)
 
     after do
       exporter.stop
+      @request_thread.kill.join # rubocop:disable RSpec/InstanceVariable -- allowing for now
     end
 
     with_them do

spec/models/members/group_member_spec.rb

@@ -12,11 +12,12 @@
   context 'scopes' do
     let_it_be(:user_1) { create(:user) }
     let_it_be(:user_2) { create(:user) }
+    let_it_be(:user_3) { create(:user) }
 
-    it 'counts users by group ID' do
-      group_1 = create(:group)
-      group_2 = create(:group)
+    let_it_be(:group_1) { create(:group) }
+    let_it_be(:group_2) { create(:group) }
 
+    it 'counts users by group ID' do
       group_1.add_owner(user_1)
       group_1.add_owner(user_2)
       group_2.add_owner(user_1)