Merge branch 'add-python-pip-config-class' into 'master'

Add config file class for Python Pip `requirements.txt`

See merge request https://gitlab.com/gitlab-org/gitlab/-/merge_requests/166633



Merged-by: Gary Holtz <gholtz@gitlab.com>
Approved-by: Moaz Khalifa <mkhalifa@gitlab.com>
Approved-by: Gary Holtz <gholtz@gitlab.com>
Reviewed-by: Moaz Khalifa <mkhalifa@gitlab.com>
Co-authored-by: lma-git <lma@gitlab.com>

doc/user/project/repository/code_suggestions/repository_xray.md

@@ -47,6 +47,7 @@ The Repository X-Ray searches a maximum of two directory levels from the reposit
 | Java       | Gradle          | `build.gradle`       | 17.4 or later  |
 | Java       | Maven           | `pom.xml`            | 17.4 or later  |
 | Kotlin     | Gradle          | `build.gradle.kts`   | 17.5 or later  |
+| Python     | Pip             | `requirements.txt`   | 17.5 or later  |
 | Ruby       | RubyGems        | `Gemfile.lock`       | 17.4 or later  |
 
 ## Enable Repository X-Ray in your CI pipeline (deprecated)

ee/lib/ai/context/dependencies/config_files/constants.rb

@@ -23,6 +23,7 @@ module Constants
             ConfigFiles::JavaGradle,
             ConfigFiles::JavaMaven,
             ConfigFiles::KotlinGradle,
+            ConfigFiles::PythonPip,
             ConfigFiles::RubyGemsLock
           ].freeze
         end

ee/lib/ai/context/dependencies/config_files/python_pip.rb

+# frozen_string_literal: true
+
+module Ai
+  module Context
+    module Dependencies
+      module ConfigFiles
+        class PythonPip < Base
+          OPTION_REGEX = /^-/
+          NAME_VERSION_REGEX = /(?<name>^[^!=><~]+)(?<version>[!=><~]+.*$)?/
+          OTHER_SPECIFIERS_REGEX = /[@;]+.*$/ # Matches URL or other non-version specifiers at the end of line
+          COMMENT_ONLY_REGEX = /^#/
+          INLINE_COMMENT_REGEX = /\s+#.*$/
+
+          def self.file_name_glob
+            'requirements.txt'
+          end
+
+          def self.lang_name
+            'Python'
+          end
+
+          private
+
+          ### Example format:
+          #
+          # requests>=2.0,<3.0      # Version range
+          # numpy==1.26.4           # Exact version match
+          # fastapi-health!=0.3.0   # Exclusion
+          #
+          # # New supported formats
+          # pytest >= 2.6.4 ; python_version < '3.8'
+          # openpyxl == 3.1.2
+          # urllib3 @ https://github.com/path/main.zip
+          #
+          # # Nested requirement files currently not supported
+          # # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/491800
+          # -r other_requirements.txt
+          # # Other options
+          # -i https://pypi.org/simple
+          # --python-version 3
+          #
+          def extract_libs
+            content.each_line.filter_map do |line|
+              line.strip!
+              next if line.blank? || Regexp.union(COMMENT_ONLY_REGEX, OPTION_REGEX).match?(line)
+
+              parse_lib(line)
+            end
+          end
+
+          def parse_lib(line)
+            line.gsub!(Regexp.union(INLINE_COMMENT_REGEX, OTHER_SPECIFIERS_REGEX), '')
+            match = NAME_VERSION_REGEX.match(line)
+
+            Lib.new(name: match[:name], version: match[:version]) if match
+          end
+        end
+      end
+    end
+  end
+end

ee/spec/lib/ai/context/dependencies/config_files/python_pip_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Ai::Context::Dependencies::ConfigFiles::PythonPip, feature_category: :code_suggestions do
+  it 'returns the expected language value' do
+    expect(described_class.lang).to eq('python')
+  end
+
+  it_behaves_like 'parsing a valid dependency config file' do
+    let(:config_file_content) do
+      <<~CONTENT
+        requests>=2.0,<3.0      # Version range
+        numpy==1.26.4           # Exact version match
+        python_dateutil>=2.5.3
+        fastapi-health!=0.3.0
+
+        # New supported formats
+        pytest >= 2.6.4 ; python_version < '3.8'
+        openpyxl == 3.1.2
+        urllib3 @ https://github.com/path/main.zip
+        requests [security] >= 2.8.1, == 2.8.*
+
+        # Nested requirement files currently not supported
+        -r other_requirements.txt
+        # Other options
+        -i https://pypi.org/simple
+        --python-version 3
+        --no-clean
+        -e .
+      CONTENT
+    end
+
+    let(:expected_formatted_lib_names) do
+      [
+        'requests (>=2.0,<3.0)',
+        'numpy (==1.26.4)',
+        'python_dateutil (>=2.5.3)',
+        'fastapi-health (!=0.3.0)',
+        'pytest (>= 2.6.4)',
+        'openpyxl (== 3.1.2)',
+        'urllib3',
+        'requests [security] (>= 2.8.1, == 2.8.*)'
+      ]
+    end
+  end
+
+  it_behaves_like 'parsing an invalid dependency config file' do
+    let(:invalid_config_file_content) { '' }
+    let(:expected_parsing_error_message) { 'file empty' }
+  end
+
+  describe '.matches?' do
+    using RSpec::Parameterized::TableSyntax
+
+    where(:path, :matches) do
+      'requirements.txt'            | true
+      'dir/requirements.txt'        | true
+      'dir/subdir/requirements.txt' | true
+      'dir/requirements'            | false
+      'xrequirements.txt'           | false
+      'Requirements.txt'            | false
+      'requirements_txt'            | false
+      'requirements'                | false
+    end
+
+    with_them do
+      it 'matches the file name glob pattern at various directory levels' do
+        expect(described_class.matches?(path)).to eq(matches)
+      end
+    end
+  end
+end