From 69960584557f540dc2b5117147214f95f57a582f Mon Sep 17 00:00:00 2001
From: Leaminn Ma <lma@gitlab.com>
Date: Mon, 23 Sep 2024 21:48:06 +0000
Subject: [PATCH] Add Python Pip config file class
Adds the Python Pip config file class which
contains logic to parse 'requirements.txt' files.
---
.../code_suggestions/repository_xray.md | 1 +
.../dependencies/config_files/constants.rb | 1 +
.../dependencies/config_files/python_pip.rb | 61 ++++++++++++++++
.../config_files/python_pip_spec.rb | 73 +++++++++++++++++++
4 files changed, 136 insertions(+)
create mode 100644 ee/lib/ai/context/dependencies/config_files/python_pip.rb
create mode 100644 ee/spec/lib/ai/context/dependencies/config_files/python_pip_spec.rb
diff --git a/doc/user/project/repository/code_suggestions/repository_xray.md b/doc/user/project/repository/code_suggestions/repository_xray.md
index 3ac5c4e4332ad..6e81c27fe111a 100644
--- a/doc/user/project/repository/code_suggestions/repository_xray.md
+++ b/doc/user/project/repository/code_suggestions/repository_xray.md
@@ -46,6 +46,7 @@ The Repository X-Ray searches a maximum of two directory levels from the reposit
| Java | Gradle | `build.gradle` | 17.4 or later |
| Java | Maven | `pom.xml` | 17.4 or later |
| Kotlin | Gradle | `build.gradle.kts` | 17.5 or later |
+| Python | Pip | `requirements.txt` | 17.5 or later |
| Ruby | RubyGems | `Gemfile.lock` | 17.4 or later |
## Enable Repository X-Ray in your CI pipeline (deprecated)
diff --git a/ee/lib/ai/context/dependencies/config_files/constants.rb b/ee/lib/ai/context/dependencies/config_files/constants.rb
index ca7dc3bd7e8ec..4c50141f2d364 100644
--- a/ee/lib/ai/context/dependencies/config_files/constants.rb
+++ b/ee/lib/ai/context/dependencies/config_files/constants.rb
@@ -22,6 +22,7 @@ module Constants
ConfigFiles::JavaGradle,
ConfigFiles::JavaMaven,
ConfigFiles::KotlinGradle,
+ ConfigFiles::PythonPip,
ConfigFiles::RubyGemsLock
].freeze
end
diff --git a/ee/lib/ai/context/dependencies/config_files/python_pip.rb b/ee/lib/ai/context/dependencies/config_files/python_pip.rb
new file mode 100644
index 0000000000000..0ffec846df13d
--- /dev/null
+++ b/ee/lib/ai/context/dependencies/config_files/python_pip.rb
@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Ai
+ module Context
+ module Dependencies
+ module ConfigFiles
+ class PythonPip < Base
+ OPTION_REGEX = /^-/
+ NAME_VERSION_REGEX = /(?<name>^[^!=><~]+)(?<version>[!=><~]+.*$)?/
+ OTHER_SPECIFIERS_REGEX = /[@;]+.*$/ # Matches URL or other non-version specifiers at the end of line
+ COMMENT_ONLY_REGEX = /^#/
+ INLINE_COMMENT_REGEX = /\s+#.*$/
+
+ def self.file_name_glob
+ 'requirements.txt'
+ end
+
+ def self.lang_name
+ 'Python'
+ end
+
+ private
+
+ ### Example format:
+ #
+ # requests>=2.0,<3.0 # Version range
+ # numpy==1.26.4 # Exact version match
+ # fastapi-health!=0.3.0 # Exclusion
+ #
+ # # New supported formats
+ # pytest >= 2.6.4 ; python_version < '3.8'
+ # openpyxl == 3.1.2
+ # urllib3 @ https://github.com/path/main.zip
+ #
+ # # Nested requirement files currently not supported
+ # # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/491800
+ # -r other_requirements.txt
+ # # Other options
+ # -i https://pypi.org/simple
+ # --python-version 3
+ #
+ def extract_libs
+ content.each_line.filter_map do |line|
+ line.strip!
+ next if line.blank? || Regexp.union(COMMENT_ONLY_REGEX, OPTION_REGEX).match?(line)
+
+ parse_lib(line)
+ end
+ end
+
+ def parse_lib(line)
+ line.gsub!(Regexp.union(INLINE_COMMENT_REGEX, OTHER_SPECIFIERS_REGEX), '')
+ match = NAME_VERSION_REGEX.match(line)
+
+ Lib.new(name: match[:name], version: match[:version]) if match
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/ee/spec/lib/ai/context/dependencies/config_files/python_pip_spec.rb b/ee/spec/lib/ai/context/dependencies/config_files/python_pip_spec.rb
new file mode 100644
index 0000000000000..f0209ab189d4c
--- /dev/null
+++ b/ee/spec/lib/ai/context/dependencies/config_files/python_pip_spec.rb
@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Ai::Context::Dependencies::ConfigFiles::PythonPip, feature_category: :code_suggestions do
+ it 'returns the expected language value' do
+ expect(described_class.lang).to eq('python')
+ end
+
+ it_behaves_like 'parsing a valid dependency config file' do
+ let(:config_file_content) do
+ <<~CONTENT
+ requests>=2.0,<3.0 # Version range
+ numpy==1.26.4 # Exact version match
+ python_dateutil>=2.5.3
+ fastapi-health!=0.3.0
+
+ # New supported formats
+ pytest >= 2.6.4 ; python_version < '3.8'
+ openpyxl == 3.1.2
+ urllib3 @ https://github.com/path/main.zip
+ requests [security] >= 2.8.1, == 2.8.*
+
+ # Nested requirement files currently not supported
+ -r other_requirements.txt
+ # Other options
+ -i https://pypi.org/simple
+ --python-version 3
+ --no-clean
+ -e .
+ CONTENT
+ end
+
+ let(:expected_formatted_lib_names) do
+ [
+ 'requests (>=2.0,<3.0)',
+ 'numpy (==1.26.4)',
+ 'python_dateutil (>=2.5.3)',
+ 'fastapi-health (!=0.3.0)',
+ 'pytest (>= 2.6.4)',
+ 'openpyxl (== 3.1.2)',
+ 'urllib3',
+ 'requests [security] (>= 2.8.1, == 2.8.*)'
+ ]
+ end
+ end
+
+ it_behaves_like 'parsing an invalid dependency config file' do
+ let(:invalid_config_file_content) { '' }
+ let(:expected_parsing_error_message) { 'file empty' }
+ end
+
+ describe '.matches?' do
+ using RSpec::Parameterized::TableSyntax
+
+ where(:path, :matches) do
+ 'requirements.txt' | true
+ 'dir/requirements.txt' | true
+ 'dir/subdir/requirements.txt' | true
+ 'dir/requirements' | false
+ 'xrequirements.txt' | false
+ 'Requirements.txt' | false
+ 'requirements_txt' | false
+ 'requirements' | false
+ end
+
+ with_them do
+ it 'matches the file name glob pattern at various directory levels' do
+ expect(described_class.matches?(path)).to eq(matches)
+ end
+ end
+ end
+end
--
GitLab