diff --git a/Gemfile b/Gemfile
index 811ad8406e6e64c77e7b546ca6ab5a84148c610b..59f1f96a85cf5a5da3d757386a7cd700864aa0ee 100644
--- a/Gemfile
+++ b/Gemfile
@@ -59,7 +59,7 @@ end
gem 'gitlab-backup-cli', path: 'gems/gitlab-backup-cli', require: 'gitlab/backup/cli', feature_category: :backup_restore
-gem 'gitlab-secret_detection', path: 'gems/gitlab-secret_detection', feature_category: :secret_detection
+gem 'gitlab-secret_detection', '< 1.0', feature_category: :secret_detection
# Responders respond_to and respond_with
gem 'responders', '~> 3.0', feature_category: :shared
diff --git a/Gemfile.checksum b/Gemfile.checksum
index 8c1f2f54e3e3287b2a3f9d18863afef95c6802fe..60982ddaa464d79b95c686700a33a27fcbbaf9bb 100644
--- a/Gemfile.checksum
+++ b/Gemfile.checksum
@@ -239,6 +239,7 @@
{"name":"gitlab-markup","version":"1.9.0","platform":"ruby","checksum":"7eda045a08ec2d110084252fa13a8c9eac8bdac0e302035ca7db4b82bcbd7ed4"},
{"name":"gitlab-net-dns","version":"0.9.2","platform":"ruby","checksum":"f726d978479d43810819f12a45c0906d775a07e34df111bbe693fffbbef3059d"},
{"name":"gitlab-sdk","version":"0.3.1","platform":"ruby","checksum":"48ba49084f4ab92df7c7ef9f347020d9dfdf6ed9c1e782b67264e98ffe6ea710"},
+{"name":"gitlab-secret_detection","version":"0.14.2","platform":"ruby","checksum":"c6d3bc92b47cdf930ff7bf1e519a849353f33df1a2b4493078963769854850f0"},
{"name":"gitlab-security_report_schemas","version":"0.1.2.min15.0.0.max15.2.1","platform":"ruby","checksum":"300037487ec9d51a814f648514ff521cb82b94fc51d9fe53389175b36ac680ae"},
{"name":"gitlab-styles","version":"13.0.2","platform":"ruby","checksum":"e662b9334643763b55a861f9e26091096547f98179bd89b0fa8d6c6fb8cec861"},
{"name":"gitlab_chronic_duration","version":"0.12.0","platform":"ruby","checksum":"0d766944d415b5c831f176871ee8625783fc0c5bfbef2d79a3a616f207ffc16d"},
diff --git a/Gemfile.lock b/Gemfile.lock
index 5bf5bac2dfe9127ba910116eaaa4e0e379540b67..8e04937b9b9a482d1b020a8d1a0d0a352be86842 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -107,16 +107,6 @@ PATH
diffy
pg_query
-PATH
- remote: gems/gitlab-secret_detection
- specs:
- gitlab-secret_detection (0.1.1)
- grpc (= 1.63.0)
- grpc-tools (= 1.63.0)
- parallel (~> 1.22)
- re2 (~> 2.7)
- toml-rb (~> 2.2)
-
PATH
remote: gems/gitlab-utils
specs:
@@ -785,6 +775,12 @@ GEM
activesupport (>= 5.2.0)
rake (~> 13.0)
snowplow-tracker (~> 0.8.0)
+ gitlab-secret_detection (0.14.2)
+ grpc (= 1.63.0)
+ grpc-tools (= 1.63.0)
+ parallel (~> 1.19)
+ re2 (= 2.7.0)
+ toml-rb (~> 2.2.0)
gitlab-security_report_schemas (0.1.2.min15.0.0.max15.2.1)
activesupport (>= 6, < 8)
json_schemer (~> 2.3.0)
@@ -2109,7 +2105,7 @@ DEPENDENCIES
gitlab-safe_request_store!
gitlab-schema-validation!
gitlab-sdk (~> 0.3.0)
- gitlab-secret_detection!
+ gitlab-secret_detection (< 1.0)
gitlab-security_report_schemas (= 0.1.2.min15.0.0.max15.2.1)
gitlab-sidekiq-fetcher!
gitlab-styles (~> 13.0.2)
diff --git a/Gemfile.next.checksum b/Gemfile.next.checksum
index 292886709ab0ba1440b0466d5c807e285cad448d..da77eb94674609bb7d06b055e2b3b8d7e54951c5 100644
--- a/Gemfile.next.checksum
+++ b/Gemfile.next.checksum
@@ -239,6 +239,7 @@
{"name":"gitlab-markup","version":"1.9.0","platform":"ruby","checksum":"7eda045a08ec2d110084252fa13a8c9eac8bdac0e302035ca7db4b82bcbd7ed4"},
{"name":"gitlab-net-dns","version":"0.9.2","platform":"ruby","checksum":"f726d978479d43810819f12a45c0906d775a07e34df111bbe693fffbbef3059d"},
{"name":"gitlab-sdk","version":"0.3.1","platform":"ruby","checksum":"48ba49084f4ab92df7c7ef9f347020d9dfdf6ed9c1e782b67264e98ffe6ea710"},
+{"name":"gitlab-secret_detection","version":"0.14.2","platform":"ruby","checksum":"c6d3bc92b47cdf930ff7bf1e519a849353f33df1a2b4493078963769854850f0"},
{"name":"gitlab-security_report_schemas","version":"0.1.2.min15.0.0.max15.2.1","platform":"ruby","checksum":"300037487ec9d51a814f648514ff521cb82b94fc51d9fe53389175b36ac680ae"},
{"name":"gitlab-styles","version":"13.0.2","platform":"ruby","checksum":"e662b9334643763b55a861f9e26091096547f98179bd89b0fa8d6c6fb8cec861"},
{"name":"gitlab_chronic_duration","version":"0.12.0","platform":"ruby","checksum":"0d766944d415b5c831f176871ee8625783fc0c5bfbef2d79a3a616f207ffc16d"},
diff --git a/Gemfile.next.lock b/Gemfile.next.lock
index 9397c74ed90abd97fcfb08b90ca52369de8b4be2..575c0952334dcd396eb834c14d484af087ddd1f2 100644
--- a/Gemfile.next.lock
+++ b/Gemfile.next.lock
@@ -107,16 +107,6 @@ PATH
diffy
pg_query
-PATH
- remote: gems/gitlab-secret_detection
- specs:
- gitlab-secret_detection (0.1.1)
- grpc (= 1.63.0)
- grpc-tools (= 1.63.0)
- parallel (~> 1.22)
- re2 (~> 2.7)
- toml-rb (~> 2.2)
-
PATH
remote: gems/gitlab-utils
specs:
@@ -797,6 +787,12 @@ GEM
activesupport (>= 5.2.0)
rake (~> 13.0)
snowplow-tracker (~> 0.8.0)
+ gitlab-secret_detection (0.14.2)
+ grpc (= 1.63.0)
+ grpc-tools (= 1.63.0)
+ parallel (~> 1.19)
+ re2 (= 2.7.0)
+ toml-rb (~> 2.2.0)
gitlab-security_report_schemas (0.1.2.min15.0.0.max15.2.1)
activesupport (>= 6, < 8)
json_schemer (~> 2.3.0)
@@ -2140,7 +2136,7 @@ DEPENDENCIES
gitlab-safe_request_store!
gitlab-schema-validation!
gitlab-sdk (~> 0.3.0)
- gitlab-secret_detection!
+ gitlab-secret_detection (< 1.0)
gitlab-security_report_schemas (= 0.1.2.min15.0.0.max15.2.1)
gitlab-sidekiq-fetcher!
gitlab-styles (~> 13.0.2)
diff --git a/ee/lib/gitlab/checks/secrets_check.rb b/ee/lib/gitlab/checks/secrets_check.rb
index 2a91df7a669c58bf309d6dfe291c73eda46a9679..bcfc8b7febff88dbe0c0352725acc15c5e05f9a0 100644
--- a/ee/lib/gitlab/checks/secrets_check.rb
+++ b/ee/lib/gitlab/checks/secrets_check.rb
@@ -4,12 +4,13 @@ module Gitlab
module Checks
class SecretsCheck < ::Gitlab::Checks::BaseBulkChecker
include Gitlab::InternalEventsTracking
+ include Gitlab::Utils::StrongMemoize
ERROR_MESSAGES = {
- failed_to_scan_regex_error: "\n - Failed to scan blob(id: %{blob_id}) due to regex error.",
- blob_timed_out_error: "\n - Scanning blob(id: %{blob_id}) timed out.",
+ failed_to_scan_regex_error: "\n - Failed to scan blob(id: %{payload_id}) due to regex error.",
+ blob_timed_out_error: "\n - Scanning blob(id: %{payload_id}) timed out.",
scan_timeout_error: 'Secret detection scan timed out.',
- scan_initialization_error: 'Secret detection scan failed to initialize.',
+ scan_initialization_error: 'Secret detection scan failed to initialize. %{error_msg}',
invalid_input_error: 'Secret detection scan failed due to invalid input.',
invalid_scan_status_code_error: 'Invalid secret detection scan status, check passed.',
too_many_tree_entries_error: 'Too many tree entries exist for commit(sha: %{sha}).'
@@ -29,9 +30,10 @@ class SecretsCheck < ::Gitlab::Checks::BaseBulkChecker
"found the following secrets in commit: %{sha}",
finding_message_occurrence_path: "\n-- %{path}:",
finding_message_occurrence_line: "%{line_number} | %{description}",
- finding_message: "\n\nSecret leaked in blob: %{blob_id}" \
+ finding_message: "\n\nSecret leaked in blob: %{payload_id}" \
"\n -- line:%{line_number} | %{description}",
- found_secrets_footer: "\n--------------------------------------------------\n\n"
+ found_secrets_footer: "\n--------------------------------------------------\n\n",
+ invalid_encoding: "Could not convert data to UTF-8 from %{encoding}"
}.freeze
PAYLOAD_BYTES_LIMIT = 1.megabyte # https://handbook.gitlab.com/handbook/engineering/architecture/design-documents/secret_detection/#target-types
@@ -39,10 +41,11 @@ class SecretsCheck < ::Gitlab::Checks::BaseBulkChecker
DOCUMENTATION_PATH = 'user/application_security/secret_detection/secret_push_protection/index.html'
DOCUMENTATION_PATH_ANCHOR = 'resolve-a-blocked-push'
EXCLUSION_TYPE_MAP = {
- path: ::Gitlab::SecretDetection::GRPC::ScanRequest::ExclusionType::EXCLUSION_TYPE_RULE,
- raw_value: ::Gitlab::SecretDetection::GRPC::ScanRequest::ExclusionType::EXCLUSION_TYPE_RAW_VALUE
+ rule: ::Gitlab::SecretDetection::GRPC::ExclusionType::EXCLUSION_TYPE_RULE,
+ path: ::Gitlab::SecretDetection::GRPC::ExclusionType::EXCLUSION_TYPE_PATH,
+ raw_value: ::Gitlab::SecretDetection::GRPC::ExclusionType::EXCLUSION_TYPE_RAW_VALUE,
+ unknown: ::Gitlab::SecretDetection::GRPC::ExclusionType::EXCLUSION_TYPE_UNSPECIFIED
}.freeze
- UNKNOWN_EXCLUSION_TYPE = ::Gitlab::SecretDetection::GRPC::ScanRequest::ExclusionType::EXCLUSION_TYPE_UNSPECIFIED
# HUNK_HEADER_REGEX matches a line starting with @@, followed by - and digits (starting line number
# and range in the original file, comma and more digits optional), then + and digits (starting line number
@@ -53,6 +56,7 @@ class SecretsCheck < ::Gitlab::Checks::BaseBulkChecker
# Maximum depth any path exclusion can have.
MAX_PATH_EXCLUSIONS_DEPTH = 20
+ # rubocop:disable Metrics/AbcSize -- This will be refactored in this epic (https://gitlab.com/groups/gitlab-org/-/epics/16376)
def validate!
# Return early and do not perform the check:
# 1. unless license is ultimate
@@ -80,9 +84,7 @@ def validate!
return
end
- if Feature.enabled?(:use_secret_detection_service, project) &&
- ::Gitlab::Saas.feature_available?(:secret_detection_service) &&
- !::Gitlab::CurrentSettings.gitlab_dedicated_instance
+ if use_secret_detection_service?
sds_host = ::Gitlab::CurrentSettings.current_application_settings.secret_detection_service_url
@sds_auth_token = ::Gitlab::CurrentSettings.current_application_settings.secret_detection_service_auth_token
@@ -101,16 +103,33 @@ def validate!
end
end
+ thread = nil
+
logger.log_timed(LOG_MESSAGES[:secrets_check]) do
# Ensure consistency between different payload types (e.g., git diffs and full files) for scanning.
payloads = standardize_payloads
- send_request_to_sds(payloads, exclusions: active_exclusions)
+ if use_secret_detection_service?
+ thread = Thread.new do
+ # This is to help identify the thread in case of a crash
+ Thread.current.name = "secrets_check"
+
+ # All the code run in the thread handles exceptions so we can leave these off
+ Thread.current.abort_on_exception = false
+ Thread.current.report_on_exception = false
+
+ send_request_to_sds(payloads, exclusions: active_exclusions)
+ end
+ end
# Pass payloads to gem for scanning.
- response = ::Gitlab::SecretDetection::Scan
- .new(logger: secret_detection_logger)
- .secrets_scan(payloads, timeout: logger.time_left, exclusions: active_exclusions)
+ response = ::Gitlab::SecretDetection::Core::Scanner
+ .new(rules: ruleset, logger: secret_detection_logger)
+ .secrets_scan(
+ payloads,
+ timeout: logger.time_left,
+ exclusions: active_exclusions
+ )
# Log audit events for exlusions that were applied.
log_applied_exclusions_audit_events(response.applied_exclusions)
@@ -119,24 +138,47 @@ def validate!
format_response(response)
# TODO: Perhaps have a separate message for each and better logging?
- rescue ::Gitlab::SecretDetection::Scan::RulesetParseError,
- ::Gitlab::SecretDetection::Scan::RulesetCompilationError => _
- secret_detection_logger.error(message: ERROR_MESSAGES[:scan_initialization_error])
+ rescue ::Gitlab::SecretDetection::Core::Scanner::RulesetParseError,
+ ::Gitlab::SecretDetection::Core::Scanner::RulesetCompilationError => e
+
+ message = format(ERROR_MESSAGES[:scan_initialization_error], { error_msg: e.message })
+ secret_detection_logger.error(message: message)
+ ensure
+ # clean up the thread
+ thread&.exit
end
end
+ # rubocop:enable Metrics/AbcSize
private
attr_reader :sds_client, :sds_auth_token
+ ##############################
+ # Helpers
+
+ def ruleset
+ ::Gitlab::SecretDetection::Core::Ruleset.new(
+ logger: secret_detection_logger
+ ).rules
+ end
+
+ strong_memoize_attr :ruleset
+
##############################
# Project Eligibility Checks
def run_pre_receive_secret_detection?
- Gitlab::CurrentSettings.current_application_settings.pre_receive_secret_detection_enabled &&
+ ::Gitlab::CurrentSettings.current_application_settings.pre_receive_secret_detection_enabled &&
project.security_setting&.pre_receive_secret_detection_enabled
end
+ def use_secret_detection_service?
+ Feature.enabled?(:use_secret_detection_service, project) &&
+ ::Gitlab::Saas.feature_available?(:secret_detection_service) &&
+ !::Gitlab::CurrentSettings.gitlab_dedicated_instance
+ end
+
###############
# Scan Checks
@@ -149,7 +191,7 @@ def use_diff_scan?
end
def includes_full_revision_history?
- Gitlab::Git.blank_ref?(changes_access.changes.first[:newrev])
+ ::Gitlab::Git.blank_ref?(changes_access.changes.first[:newrev])
end
def skip_secret_detection_commit_message?
@@ -188,10 +230,18 @@ def log_applied_exclusions_audit_events(applied_exclusions)
# scanning of either `rule` or `raw_value` type. For `path` exclusions, we create the audit events
# when applied while formatting the response.
applied_exclusions.each do |exclusion|
- log_exclusion_audit_event(exclusion)
+ project_security_exclusion = get_project_security_exclusion_from_sds_exclusion(exclusion)
+ log_exclusion_audit_event(project_security_exclusion) unless project_security_exclusion.nil?
end
end
+ def get_project_security_exclusion_from_sds_exclusion(exclusion)
+ return exclusion if exclusion.is_a?(::Security::ProjectSecurityExclusion)
+
+ # TODO When we implement 2-way SDS communication, we should add the type to this lookup
+ project.security_exclusions.where(value: exclusion.value).first # rubocop:disable CodeReuse/ActiveRecord -- Need to be able to link GRPC::Exclusion to ProjectSecurityExclusion
+ end
+
def log_exclusion_audit_event(exclusion)
audit_context = {
name: 'project_security_exclusion_applied',
@@ -243,7 +293,8 @@ def standardize_payloads
payloads = get_diffs
payloads = payloads.flat_map do |payload|
- parse_diffs(payload)
+ p_ary = parse_diffs(payload)
+ build_payloads(p_ary)
end
payloads.compact.empty? ? nil : payloads
@@ -256,11 +307,13 @@ def standardize_payloads
payloads.reject! { |payload| payload.size > PAYLOAD_BYTES_LIMIT || payload.binary }
payloads.map do |payload|
- {
- id: payload.id,
- data: payload.data,
- offset: 1
- }
+ build_payload(
+ {
+ id: payload.id,
+ data: payload.data,
+ offset: 1
+ }
+ )
end
end
end
@@ -415,8 +468,8 @@ def revisions
def format_response(response)
# Try to retrieve file path and commit sha for the diffs found.
if [
- ::Gitlab::SecretDetection::Status::FOUND,
- ::Gitlab::SecretDetection::Status::FOUND_WITH_ERRORS
+ ::Gitlab::SecretDetection::Core::Status::FOUND,
+ ::Gitlab::SecretDetection::Core::Status::FOUND_WITH_ERRORS
].include?(response.status)
results = transform_findings(response)
@@ -426,17 +479,17 @@ def format_response(response)
end
case response.status
- when ::Gitlab::SecretDetection::Status::NOT_FOUND
+ when ::Gitlab::SecretDetection::Core::Status::NOT_FOUND
# No secrets found, we log and skip the check.
secret_detection_logger.info(message: LOG_MESSAGES[:secrets_not_found])
- when ::Gitlab::SecretDetection::Status::FOUND
+ when ::Gitlab::SecretDetection::Core::Status::FOUND
# One or more secrets found, generate message with findings and fail check.
message = build_secrets_found_message(results)
secret_detection_logger.info(message: LOG_MESSAGES[:found_secrets])
raise ::Gitlab::GitAccess::ForbiddenError, message
- when ::Gitlab::SecretDetection::Status::FOUND_WITH_ERRORS
+ when ::Gitlab::SecretDetection::Core::Status::FOUND_WITH_ERRORS
# One or more secrets found, but with scan errors, so we
# generate a message with findings and errors, and fail the check.
message = build_secrets_found_message(results, with_errors: true)
@@ -444,12 +497,12 @@ def format_response(response)
secret_detection_logger.info(message: LOG_MESSAGES[:found_secrets_with_errors])
raise ::Gitlab::GitAccess::ForbiddenError, message
- when ::Gitlab::SecretDetection::Status::SCAN_TIMEOUT
+ when ::Gitlab::SecretDetection::Core::Status::SCAN_TIMEOUT
# Entire scan timed out, we log and skip the check for now.
secret_detection_logger.error(message: ERROR_MESSAGES[:scan_timeout_error])
- when ::Gitlab::SecretDetection::Status::INPUT_ERROR
- # Scan failed due to invalid input. We skip the check because an input error
- # could be due to not having `diffs` being empty (i.e. no new diffs to scan).
+ when ::Gitlab::SecretDetection::Core::Status::INPUT_ERROR
+ # Scan failed due to invalid input. We skip the check because of an input error
+ # which could be due to not having anything to scan.
secret_detection_logger.error(message: ERROR_MESSAGES[:invalid_input_error])
else
# Invalid status returned by the scanning service/gem, we don't
@@ -496,7 +549,7 @@ def build_secrets_found_message(results, with_errors: false)
def build_finding_message(finding, type)
case finding.status
- when ::Gitlab::SecretDetection::Status::FOUND
+ when ::Gitlab::SecretDetection::Core::Status::FOUND
track_secret_found(finding.description)
case type
@@ -505,9 +558,9 @@ def build_finding_message(finding, type)
when :blob
build_blob_finding_message(finding)
end
- when ::Gitlab::SecretDetection::Status::SCAN_ERROR
+ when ::Gitlab::SecretDetection::Core::Status::SCAN_ERROR
format(ERROR_MESSAGES[:failed_to_scan_regex_error], finding.to_h)
- when ::Gitlab::SecretDetection::Status::PAYLOAD_TIMEOUT
+ when ::Gitlab::SecretDetection::Core::Status::PAYLOAD_TIMEOUT
format(ERROR_MESSAGES[:blob_timed_out_error], finding.to_h)
end
end
@@ -529,7 +582,7 @@ def build_blob_finding_message(finding)
# rubocop:disable Metrics/CyclomaticComplexity -- Not easy to move complexity away into other methods, entire method will be refactored shortly.
def transform_findings(response)
# Let's group the findings by the blob id.
- findings_by_blobs = response.results.group_by(&:blob_id)
+ findings_by_blobs = response.results.group_by(&:payload_id)
# We create an empty hash for the structure we'll create later as we pull out tree entries.
findings_by_commits = {}
@@ -572,7 +625,7 @@ def transform_findings(response)
# is available, we will likely be able move this check to the gem/secret detection service
# since paths will be available pre-scanning.
if matches_excluded_path?(entry.path)
- response.results.delete_if { |finding| finding.blob_id == entry.id }
+ response.results.delete_if { |finding| finding.payload_id == entry.id }
findings_by_blobs.delete(entry.id)
@@ -596,7 +649,7 @@ def transform_findings(response)
end
# Remove blobs that has already been found in a tree entry.
- findings_by_blobs.delete_if { |blob_id, _| blobs_found_with_tree_entries.include?(blob_id) }
+ findings_by_blobs.delete_if { |payload_id, _| blobs_found_with_tree_entries.include?(payload_id) }
# Return the findings as a hash sorted by commits and blobs (minus ones already found).
{
@@ -652,30 +705,61 @@ def send_request_to_sds(payloads, exclusions: {})
secret_detection_logger.error(message: e.message)
end
- def build_sds_request(data, exclusions: {}, tags: [])
- payloads = data.inject([]) do |payloads, datum|
- payloads << ::Gitlab::SecretDetection::GRPC::ScanRequest::Payload.new(
- id: datum[:id],
- data: datum[:data]
- )
+ # Expects an array of either Hashes or ScanRequest::Payloads
+ def build_payloads(data)
+ data.inject([]) do |payloads, datum|
+ payloads << build_payload(datum)
+ end.compact
+ end
+
+ # Expect `payload` is a hash or GRPC::ScanRequest::Payload object
+ def build_payload(datum)
+ return datum if datum.is_a?(::Gitlab::SecretDetection::GRPC::ScanRequest::Payload)
+
+ original_encoding = datum[:data].encoding
+
+ unless original_encoding == Encoding::UTF_8
+ datum[:data] = datum[:data].dup.force_encoding('UTF-8') # Incident 19090 (https://gitlab.com/gitlab-com/gl-infra/production/-/issues/19090)
end
+ unless datum[:data].valid_encoding?
+ log_msg = format(LOG_MESSAGES[:invalid_encoding], { encoding: original_encoding })
+ secret_detection_logger.warn(message: log_msg)
+ return
+ end
+
+ ::Gitlab::SecretDetection::GRPC::ScanRequest::Payload.new(
+ id: datum[:id],
+ data: datum[:data], # Incident 19090 (https://gitlab.com/gitlab-com/gl-infra/production/-/issues/19090)
+ offset: datum.fetch(:offset, nil)
+ )
+ end
+
+ # Build the list of gRPC Exclusion objects
+ def build_exclusions(exclusions: {})
exclusion_ary = []
# exclusions are a hash of {string, array} pairs where the keys
# are exclusion types like raw_value or path
exclusions.each_key do |key|
exclusions[key].inject(exclusion_ary) do |array, exclusion|
- type = EXCLUSION_TYPE_MAP[exclusion.type.to_sym] || UNKNOWN_EXCLUSION_TYPE
+ type = EXCLUSION_TYPE_MAP.fetch(exclusion.type.to_sym, EXCLUSION_TYPE_MAP[:unknown])
- array << ::Gitlab::SecretDetection::GRPC::ScanRequest::Exclusion.new(
+ array << ::Gitlab::SecretDetection::GRPC::Exclusion.new(
exclusion_type: type,
value: exclusion.value
)
end
end
- Gitlab::SecretDetection::GRPC::ScanRequest.new(
+ exclusion_ary
+ end
+
+ # Puts the entire gRPC request object together
+ def build_sds_request(payloads, exclusions: {}, tags: [])
+ exclusion_ary = build_exclusions(exclusions:)
+
+ ::Gitlab::SecretDetection::GRPC::ScanRequest.new(
payloads: payloads,
exclusions: exclusion_ary,
tags: tags
diff --git a/ee/spec/lib/gitlab/checks/secrets_check_spec.rb b/ee/spec/lib/gitlab/checks/secrets_check_spec.rb
index b8197ae83c5b59951bde46cdeb4481651579b556..55e15250fbb7d22c3f85a2c93fb8cb5e4e381f76 100644
--- a/ee/spec/lib/gitlab/checks/secrets_check_spec.rb
+++ b/ee/spec/lib/gitlab/checks/secrets_check_spec.rb
@@ -43,7 +43,37 @@
stub_licensed_features(pre_receive_secret_detection: true)
end
- it_behaves_like "skips sending requests to the SDS"
+ context 'when SDS should be called (on SaaS)' do
+ before do
+ stub_saas_features(secret_detection_service: true)
+ stub_application_setting(secret_detection_service_url: 'https://example.com')
+ end
+
+ context 'when instance is Dedicated (temporarily not using SDS)' do
+ before do
+ stub_application_setting(gitlab_dedicated_instance: true)
+ end
+
+ it_behaves_like 'skips sending requests to the SDS'
+ end
+
+ context 'when instance is GitLab.com' do
+ # this is the happy path (as FFs are enabled by default)
+ it_behaves_like 'sends requests to the SDS'
+
+ context 'when `use_secret_detection_service` feature flag is disabled' do
+ before do
+ stub_feature_flags(use_secret_detection_service: false)
+ end
+
+ it_behaves_like 'skips sending requests to the SDS'
+ end
+ end
+ end
+
+ context 'when SDS should not be called (Self-Managed)' do
+ it_behaves_like 'skips sending requests to the SDS'
+ end
context 'when deleting the branch' do
# We instantiate the described class with delete_changes_access object to ensure
@@ -53,14 +83,6 @@
it_behaves_like 'skips the push check'
end
- context 'when SDS URL is defined' do
- before do
- stub_application_setting(secret_detection_service_url: 'https://example.com')
- end
-
- it_behaves_like 'skips sending requests to the SDS'
- end
-
context 'when the spp_scan_diffs flag is disabled' do
before do
stub_feature_flags(spp_scan_diffs: false)
@@ -76,11 +98,13 @@
it_behaves_like 'scan skipped when a commit has special bypass flag'
it_behaves_like 'scan skipped when secret_push_protection.skip_all push option is passed'
it_behaves_like 'scan discarded secrets because they match exclusions'
+ it_behaves_like 'detects secrets with special characters in full files'
end
context 'when the spp_scan_diffs flag is enabled' do
it_behaves_like 'diff scan passed'
it_behaves_like 'scan detected secrets in diffs'
+ it_behaves_like 'detects secrets with special characters in diffs'
context 'when the protocol is web' do
subject(:secrets_check) { described_class.new(changes_access_web) }
@@ -95,46 +119,84 @@
it_behaves_like 'scan skipped when a commit has special bypass flag'
it_behaves_like 'scan skipped when secret_push_protection.skip_all push option is passed'
it_behaves_like 'scan discarded secrets because they match exclusions'
+ it_behaves_like 'detects secrets with special characters in full files'
end
context 'when the spp_scan_diffs flag is enabled' do
it_behaves_like 'diff scan passed'
it_behaves_like 'scan detected secrets in diffs'
it_behaves_like 'processes hunk headers'
+ it_behaves_like 'detects secrets with special characters in diffs'
context 'when the protocol is web' do
subject(:secrets_check) { described_class.new(changes_access_web) }
it_behaves_like 'entire file scan passed'
it_behaves_like 'scan detected secrets'
+ it_behaves_like 'detects secrets with special characters in full files'
end
end
end
+ end
+ end
+ end
+ end
- context 'when SDS URL is set' do
- before do
- stub_application_setting(secret_detection_service_url: 'https://example.com')
- end
+ # While we prefer not to test private methods directly, the structure of the shared examples
+ # makes testing this code difficult and time-sonsuming.
+ # Remove this if refactoring the shared exazmples makes this easier through testing public methods
+ describe '#get_project_security_exclusion_from_sds_exclusion' do
+ let_it_be(:project) { create(:project) }
+ let_it_be(:pse) { create(:project_security_exclusion, :with_rule, project: project) }
+
+ let(:sds_exclusion) do
+ Gitlab::SecretDetection::GRPC::Exclusion.new(
+ exclusion_type: Gitlab::SecretDetection::GRPC::ExclusionType::EXCLUSION_TYPE_RULE,
+ value: pse.value
+ )
+ end
- it_behaves_like 'skips sending requests to the SDS'
+ it 'returns the same object if it is a ProjectSecurityExclusion' do
+ result = secrets_check.send(:get_project_security_exclusion_from_sds_exclusion, pse)
+ expect(result).to be pse
+ end
- context 'when instance is GitLab.com' do
- before do
- stub_saas_features(secret_detection_service: true)
- end
+ it 'returns the ProjectSecurityExclusion with the same value' do
+ result = secrets_check.send(:get_project_security_exclusion_from_sds_exclusion, sds_exclusion)
+ expect(result).to eq pse
+ end
+ end
- it_behaves_like 'sends requests to the SDS'
+ # Most of the shared examples exercise build_payload normally, but this tests it specifically for
+ # a situation where the data is not a valid utf8 string after being forced into one.
+ # Remove this if refactoring the shared exazmples makes this easier through testing public methods
+ describe '#build_payload' do
+ context 'when data has invalid encoding' do
+ let(:datum_id) { 'test-blob-id' }
+ let(:datum_offset) { 1 }
+ let(:original_encoding) { 'ASCII-8BIT' }
+
+ let(:data_content) { +'encoded string' }
+
+ let(:invalid_datum) do
+ {
+ id: datum_id,
+ data: data_content,
+ offset: datum_offset
+ }
+ end
- context 'when `use_secret_detection_service` is disabled`' do
- before do
- stub_feature_flags(use_secret_detection_service: false)
- end
+ it 'returns nil and logs a warning' do
+ expect(data_content).to receive(:encoding).and_return(original_encoding)
+ expect(data_content).to receive(:dup).and_return(data_content)
+ expect(data_content).to receive(:force_encoding).and_return(data_content)
+ expect(data_content).to receive(:valid_encoding?).and_return(false)
- it_behaves_like 'skips sending requests to the SDS'
- end
- end
- end
- end
+ expect(secret_detection_logger).to receive(:warn)
+ .with(message: format(log_messages[:invalid_encoding], { encoding: original_encoding }))
+
+ result = secrets_check.send(:build_payload, invalid_datum)
+ expect(result).to be_nil
end
end
end
diff --git a/ee/spec/support/shared_contexts/secrets_check_shared_contexts.rb b/ee/spec/support/shared_contexts/secrets_check_shared_contexts.rb
index 66785fd33094bfd361b3fbe1a02c8ab54340aaaa..c1b89f0edf20864f8b847b5a3e54c43733cce79d 100644
--- a/ee/spec/support/shared_contexts/secrets_check_shared_contexts.rb
+++ b/ee/spec/support/shared_contexts/secrets_check_shared_contexts.rb
@@ -34,8 +34,21 @@
let(:existing_blob) { have_attributes(class: Gitlab::Git::Blob, id: existing_blob_reference, size: 23) }
let(:new_blob) { have_attributes(class: Gitlab::Git::Blob, id: new_blob_reference, size: 24) }
- let(:existing_payload) { { id: existing_blob_reference, data: "Documentation goes here", offset: 1 } }
- let(:new_payload) { { id: new_blob_reference, data: "BASE_URL=https://foo.bar", offset: 1 } }
+ let(:existing_payload) do
+ Gitlab::SecretDetection::GRPC::ScanRequest::Payload.new(
+ id: existing_blob_reference,
+ data: "Documentation goes here",
+ offset: 1
+ )
+ end
+
+ let(:new_payload) do
+ Gitlab::SecretDetection::GRPC::ScanRequest::Payload.new(
+ id: new_blob_reference,
+ data: "BASE_URL=https://foo.bar",
+ offset: 1
+ )
+ end
let(:changes) do
[
@@ -156,11 +169,11 @@
# Error messsages with formatting
let(:failed_to_scan_regex_error) do
- format(error_messages[:failed_to_scan_regex_error], { blob_id: failed_to_scan_blob_reference })
+ format(error_messages[:failed_to_scan_regex_error], { payload_id: failed_to_scan_blob_reference })
end
let(:blob_timed_out_error) do
- format(error_messages[:blob_timed_out_error], { blob_id: timed_out_blob_reference })
+ format(error_messages[:blob_timed_out_error], { payload_id: timed_out_blob_reference })
end
let(:too_many_tree_entries_error) do
@@ -270,7 +283,7 @@
format(
log_messages[:finding_message],
{
- blob_id: new_blob_reference,
+ payload_id: new_blob_reference,
line_number: finding_line_number,
description: finding_description
}
@@ -314,6 +327,34 @@
end
end
+# In response to Incident 19090 (https://gitlab.com/gitlab-com/gl-infra/production/-/issues/19090)
+RSpec.shared_context 'special characters table' do
+ using RSpec::Parameterized::TableSyntax
+
+ where(:special_character, :description) do
+ (+'—').force_encoding('ASCII-8BIT') | 'em-dash'
+ (+'â„¢').force_encoding('ASCII-8BIT') | 'trademark'
+ (+'☀').force_encoding('ASCII-8BIT') | 'sun'
+ (+'♫').force_encoding('ASCII-8BIT') | 'beamed eighth notes'
+ (+'âš¡').force_encoding('ASCII-8BIT') | 'high voltage sign'
+ (+'âš”').force_encoding('ASCII-8BIT') | 'crossed swords'
+ (+'âš–').force_encoding('ASCII-8BIT') | 'scales'
+ (+'âš›').force_encoding('ASCII-8BIT') | 'atom symbol'
+ (+'⚜').force_encoding('ASCII-8BIT') | 'fleur-de-lis'
+ (+'âš½').force_encoding('ASCII-8BIT') | 'soccer ball'
+ (+'⛄').force_encoding('ASCII-8BIT') | 'snowman without snow'
+ (+'â›…').force_encoding('ASCII-8BIT') | 'sun behind cloud'
+ (+'⛎').force_encoding('ASCII-8BIT') | 'ophiuchus'
+ (+'â›”').force_encoding('ASCII-8BIT') | 'no entry'
+ (+'⛪').force_encoding('ASCII-8BIT') | 'church'
+ (+'⛵').force_encoding('ASCII-8BIT') | 'sailboat'
+ (+'⛺').force_encoding('ASCII-8BIT') | 'tent'
+ (+'⛽').force_encoding('ASCII-8BIT') | 'fuel pump'
+ (+'✈').force_encoding('ASCII-8BIT') | 'airplane'
+ (+'â„').force_encoding('ASCII-8BIT') | 'snowflake'
+ end
+end
+
def create_commit(blobs, message = 'Add a file')
commit = repository.commit_files(
user,
diff --git a/ee/spec/support/shared_examples/lib/gitlab/secrets_check_shared_examples.rb b/ee/spec/support/shared_examples/lib/gitlab/secrets_check_shared_examples.rb
index b9bd7f35330b32ea5995bdeaa25cf1a591ff520f..f63c3707799dbfe8cff66a03e0afeb671ecca789 100644
--- a/ee/spec/support/shared_examples/lib/gitlab/secrets_check_shared_examples.rb
+++ b/ee/spec/support/shared_examples/lib/gitlab/secrets_check_shared_examples.rb
@@ -48,13 +48,14 @@
let(:payload) do
::Gitlab::SecretDetection::GRPC::ScanRequest::Payload.new(
id: 'da66bef46dbf0ad7fdcbeec97c9eaa24c2846dda',
- data: 'BASE_URL=https://foo.bar'
+ data: 'BASE_URL=https://foo.bar',
+ offset: 1
)
end
let(:exclusion) do
- ::Gitlab::SecretDetection::GRPC::ScanRequest::Exclusion.new(
- exclusion_type: ::Gitlab::SecretDetection::GRPC::ScanRequest::ExclusionType::EXCLUSION_TYPE_RULE,
+ ::Gitlab::SecretDetection::GRPC::Exclusion.new(
+ exclusion_type: ::Gitlab::SecretDetection::GRPC::ExclusionType::EXCLUSION_TYPE_PATH,
value: 'file-exclusion-1.rb'
)
end
@@ -122,7 +123,11 @@
RSpec.shared_examples 'entire file scan passed' do
include_context 'secrets check context'
- let(:passed_scan_response) { ::Gitlab::SecretDetection::Response.new(Gitlab::SecretDetection::Status::NOT_FOUND) }
+ let(:passed_scan_response) do
+ ::Gitlab::SecretDetection::Core::Response.new(
+ status: ::Gitlab::SecretDetection::Core::Status::NOT_FOUND
+ )
+ end
context 'with quarantine directory' do
include_context 'quarantine directory exists'
@@ -130,7 +135,7 @@
it 'lists all blobs of a repository' do
expect(repository).to receive(:list_all_blobs)
.with(
- bytes_limit: Gitlab::Checks::SecretsCheck::PAYLOAD_BYTES_LIMIT + 1,
+ bytes_limit: ::Gitlab::Checks::SecretsCheck::PAYLOAD_BYTES_LIMIT + 1,
dynamic_timeout: kind_of(Float),
ignore_alternate_object_directories: true
)
@@ -170,7 +175,7 @@
expect(repository).to receive(:list_blobs)
.with(
['--not', '--all', '--not'] + changes.pluck(:newrev),
- bytes_limit: Gitlab::Checks::SecretsCheck::PAYLOAD_BYTES_LIMIT + 1,
+ bytes_limit: ::Gitlab::Checks::SecretsCheck::PAYLOAD_BYTES_LIMIT + 1,
with_paths: false,
dynamic_timeout: kind_of(Float)
)
@@ -187,7 +192,7 @@
end
it 'scans diffs' do
- expect_next_instance_of(::Gitlab::SecretDetection::Scan) do |instance|
+ expect_next_instance_of(::Gitlab::SecretDetection::Core::Scanner) do |instance|
expect(instance).to receive(:secrets_scan)
.with(
[new_payload],
@@ -217,12 +222,34 @@
RSpec.shared_examples 'diff scan passed' do
include_context 'secrets check context'
- let(:passed_scan_response) { ::Gitlab::SecretDetection::Response.new(Gitlab::SecretDetection::Status::NOT_FOUND) }
- let(:new_payload) { { id: new_blob_reference, data: "BASE_URL=https://foo.bar", offset: 1 } }
+ let(:passed_scan_response) do
+ ::Gitlab::SecretDetection::Core::Response.new(
+ status: ::Gitlab::SecretDetection::Core::Status::NOT_FOUND
+ )
+ end
+
+ # Array of hashes returned by `parse_diffs`
+ let(:raw_payloads) do
+ [
+ {
+ id: new_blob_reference,
+ data: "BASE_URL=https://foo.bar",
+ offset: 1
+ }
+ ]
+ end
+
+ let(:new_payload) do
+ ::Gitlab::SecretDetection::GRPC::ScanRequest::Payload.new(
+ id: new_blob_reference,
+ data: "BASE_URL=https://foo.bar",
+ offset: 1
+ )
+ end
let(:diff_blob) do
- Gitlab::GitalyClient::DiffBlob.new(
- left_blob_id: Gitlab::Git::SHA1_BLANK_SHA,
+ ::Gitlab::GitalyClient::DiffBlob.new(
+ left_blob_id: ::Gitlab::Git::SHA1_BLANK_SHA,
right_blob_id: new_blob_reference,
patch: "@@ -0,0 +1 @@\n+BASE_URL=https://foo.bar\n\\ No newline at end of file\n",
status: :STATUS_END_OF_PATCH,
@@ -241,7 +268,7 @@
expect(instance).to receive(:parse_diffs)
.with(diff_blob)
.once
- .and_return(new_payload)
+ .and_return(raw_payloads)
.and_call_original
end
@@ -253,7 +280,7 @@
end
it 'scans diffs' do
- expect_next_instance_of(::Gitlab::SecretDetection::Scan) do |instance|
+ expect_next_instance_of(::Gitlab::SecretDetection::Core::Scanner) do |instance|
expect(instance).to receive(:secrets_scan)
.with(
[new_payload],
@@ -284,12 +311,12 @@
include_context 'secrets check context'
let(:successful_scan_response) do
- ::Gitlab::SecretDetection::Response.new(
- Gitlab::SecretDetection::Status::FOUND,
- [
- Gitlab::SecretDetection::Finding.new(
+ ::Gitlab::SecretDetection::Core::Response.new(
+ status: ::Gitlab::SecretDetection::Core::Status::FOUND,
+ results: [
+ ::Gitlab::SecretDetection::Core::Finding.new(
new_blob_reference,
- Gitlab::SecretDetection::Status::FOUND,
+ ::Gitlab::SecretDetection::Core::Status::FOUND,
1,
"gitlab_personal_access_token",
"GitLab Personal Access Token"
@@ -300,10 +327,14 @@
let(:new_blob_reference) { 'fe29d93da4843da433e62711ace82db601eb4f8f' }
let(:new_payload) do
- { data: "SECRET=glpat-JUST20LETTERSANDNUMB", id: new_blob_reference, offset: 1 } # gitleaks:allow
+ ::Gitlab::SecretDetection::GRPC::ScanRequest::Payload.new(
+ data: "SECRET=glpat-JUST20LETTERSANDNUMB", # gitleaks:allow
+ id: new_blob_reference,
+ offset: 1
+ )
end
- let(:new_blob) { have_attributes(class: Gitlab::Git::Blob, id: new_blob_reference, size: 33) }
+ let(:new_blob) { have_attributes(class: ::Gitlab::Git::Blob, id: new_blob_reference, size: 33) }
# The new commit must have a secret, so create a commit with one.
let_it_be(:new_commit) { create_commit('.env' => 'SECRET=glpat-JUST20LETTERSANDNUMB') } # gitleaks:allow
@@ -321,7 +352,7 @@
it 'lists all blobs of a repository' do
expect(repository).to receive(:list_all_blobs)
.with(
- bytes_limit: Gitlab::Checks::SecretsCheck::PAYLOAD_BYTES_LIMIT + 1,
+ bytes_limit: ::Gitlab::Checks::SecretsCheck::PAYLOAD_BYTES_LIMIT + 1,
dynamic_timeout: kind_of(Float),
ignore_alternate_object_directories: true
)
@@ -361,7 +392,7 @@
expect(repository).to receive(:list_blobs)
.with(
['--not', '--all', '--not'] + changes.pluck(:newrev),
- bytes_limit: Gitlab::Checks::SecretsCheck::PAYLOAD_BYTES_LIMIT + 1,
+ bytes_limit: ::Gitlab::Checks::SecretsCheck::PAYLOAD_BYTES_LIMIT + 1,
with_paths: false,
dynamic_timeout: kind_of(Float)
)
@@ -378,7 +409,7 @@
end
it 'scans diffs' do
- expect_next_instance_of(::Gitlab::SecretDetection::Scan) do |instance|
+ expect_next_instance_of(::Gitlab::SecretDetection::Core::Scanner) do |instance|
expect(instance).to receive(:secrets_scan)
.with(
[new_payload],
@@ -421,12 +452,16 @@
let(:new_blob_reference) { 'fe29d93da4843da433e62711ace82db601eb4f8f' }
let(:new_payload) do
- { data: "SECRET=glpat-JUST20LETTERSANDNUMB", id: new_blob_reference, offset: 1 } # gitleaks:allow
+ ::Gitlab::SecretDetection::GRPC::ScanRequest::Payload.new(
+ data: "SECRET=glpat-JUST20LETTERSANDNUMB", # gitleaks:allow
+ id: new_blob_reference,
+ offset: 1
+ )
end
let(:tree_entries) do
[
- Gitlab::Git::Tree.new(
+ ::Gitlab::Git::Tree.new(
id: new_blob_reference,
type: :blob,
mode: '100644',
@@ -435,7 +470,7 @@
flat_path: '.env',
commit_id: new_commit
),
- Gitlab::Git::Tree.new(
+ ::Gitlab::Git::Tree.new(
id: new_blob_reference,
type: :blob,
mode: '100644',
@@ -456,27 +491,28 @@
let(:commits) do
Commit.decorate(
[
- Gitlab::Git::Commit.find(repository, new_commit),
- Gitlab::Git::Commit.find(repository, commit_with_same_blob)
+ ::Gitlab::Git::Commit.find(repository, new_commit),
+ ::Gitlab::Git::Commit.find(repository, commit_with_same_blob)
],
project
)
end
let(:successful_with_same_blob_in_multiple_commits_scan_response) do
- ::Gitlab::SecretDetection::Response.new(
- Gitlab::SecretDetection::Status::FOUND,
+ ::Gitlab::SecretDetection::Core::Response.new(
+ status: ::Gitlab::SecretDetection::Core::Status::FOUND,
+ results:
[
- Gitlab::SecretDetection::Finding.new(
+ ::Gitlab::SecretDetection::Core::Finding.new(
new_blob_reference,
- Gitlab::SecretDetection::Status::FOUND,
+ ::Gitlab::SecretDetection::Core::Status::FOUND,
1,
"gitlab_personal_access_token",
"GitLab Personal Access Token"
),
- Gitlab::SecretDetection::Finding.new(
+ ::Gitlab::SecretDetection::Core::Finding.new(
new_blob_reference,
- Gitlab::SecretDetection::Status::FOUND,
+ ::Gitlab::SecretDetection::Core::Status::FOUND,
1,
"gitlab_personal_access_token",
"GitLab Personal Access Token"
@@ -492,7 +528,7 @@
end
it 'displays the findings with their corresponding commit sha/file path' do
- expect_next_instance_of(::Gitlab::SecretDetection::Scan) do |instance|
+ expect_next_instance_of(::Gitlab::SecretDetection::Core::Scanner) do |instance|
expect(instance).to receive(:secrets_scan)
.with(
[new_payload],
@@ -532,7 +568,7 @@
end
it 'scans diffs' do
- expect_next_instance_of(::Gitlab::SecretDetection::Scan) do |instance|
+ expect_next_instance_of(::Gitlab::SecretDetection::Core::Scanner) do |instance|
expect(instance).to receive(:secrets_scan)
.with(
[new_payload],
@@ -563,7 +599,7 @@
end
it 'scans diffs' do
- expect_next_instance_of(::Gitlab::SecretDetection::Scan) do |instance|
+ expect_next_instance_of(::Gitlab::SecretDetection::Core::Scanner) do |instance|
expect(instance).to receive(:secrets_scan)
.with(
[new_payload],
@@ -666,7 +702,7 @@
let(:finding_path) { 'config/.env' }
let(:tree_entries) do
[
- Gitlab::Git::Tree.new(
+ ::Gitlab::Git::Tree.new(
id: new_blob_reference,
type: :blob,
mode: '100644',
@@ -710,24 +746,28 @@
let(:new_blob_reference) { '59ef300b246861163ee1e2ab4146e16144e4770f' }
let(:new_payload) do
- { data: "SECRET=glpat-JUST20LETTERSANDNUMB\nTOKEN=glpat-JUST20LETTERSANDNUMB", # gitleaks:allow
- id: new_blob_reference, offset: 1 }
+ ::Gitlab::SecretDetection::GRPC::ScanRequest::Payload.new(
+ data: "SECRET=glpat-JUST20LETTERSANDNUMB\nTOKEN=glpat-JUST20LETTERSANDNUMB", # gitleaks:allow
+ id: new_blob_reference,
+ offset: 1
+ )
end
let(:successful_with_multiple_findings_scan_response) do
- ::Gitlab::SecretDetection::Response.new(
- Gitlab::SecretDetection::Status::FOUND,
+ ::Gitlab::SecretDetection::Core::Response.new(
+ status: ::Gitlab::SecretDetection::Core::Status::FOUND,
+ results:
[
- Gitlab::SecretDetection::Finding.new(
+ ::Gitlab::SecretDetection::Core::Finding.new(
new_blob_reference,
- Gitlab::SecretDetection::Status::FOUND,
+ ::Gitlab::SecretDetection::Core::Status::FOUND,
1,
"gitlab_personal_access_token",
"GitLab Personal Access Token"
),
- Gitlab::SecretDetection::Finding.new(
+ ::Gitlab::SecretDetection::Core::Finding.new(
new_blob_reference,
- Gitlab::SecretDetection::Status::FOUND,
+ ::Gitlab::SecretDetection::Core::Status::FOUND,
2,
"gitlab_personal_access_token",
"GitLab Personal Access Token"
@@ -737,7 +777,7 @@
end
it 'displays all findings with their corresponding commit sha/filepath' do
- expect_next_instance_of(::Gitlab::SecretDetection::Scan) do |instance|
+ expect_next_instance_of(::Gitlab::SecretDetection::Core::Scanner) do |instance|
expect(instance).to receive(:secrets_scan)
.with(
[new_payload],
@@ -773,26 +813,30 @@
let(:new_blob_reference) { '13a31e7c93bbe8781f341e24e8ef26ef717d0da2' }
let(:new_payload) do
- { data: "SECRET=glpat-JUST20LETTERSANDNUMB;TOKEN=GR1348941JUST20LETTERSANDNUMB", # gitleaks:allow
- id: new_blob_reference, offset: 1 }
+ ::Gitlab::SecretDetection::GRPC::ScanRequest::Payload.new(
+ data: "SECRET=glpat-JUST20LETTERSANDNUMB;TOKEN=GR1348941JUST20LETTERSANDNUMB", # gitleaks:allow
+ id: new_blob_reference,
+ offset: 1
+ )
end
let(:second_finding_description) { 'GitLab Runner Registration Token' }
let(:successful_with_multiple_findings_on_same_line_scan_response) do
- ::Gitlab::SecretDetection::Response.new(
- Gitlab::SecretDetection::Status::FOUND,
+ ::Gitlab::SecretDetection::Core::Response.new(
+ status: ::Gitlab::SecretDetection::Core::Status::FOUND,
+ results:
[
- Gitlab::SecretDetection::Finding.new(
+ ::Gitlab::SecretDetection::Core::Finding.new(
new_blob_reference,
- Gitlab::SecretDetection::Status::FOUND,
+ ::Gitlab::SecretDetection::Core::Status::FOUND,
1,
"gitlab_personal_access_token",
"GitLab Personal Access Token"
),
- Gitlab::SecretDetection::Finding.new(
+ ::Gitlab::SecretDetection::Core::Finding.new(
new_blob_reference,
- Gitlab::SecretDetection::Status::FOUND,
+ ::Gitlab::SecretDetection::Core::Status::FOUND,
1,
"gitlab_runner_registration_token",
"GitLab Runner Registration Token"
@@ -802,7 +846,7 @@
end
it 'displays the findings with their corresponding commit sha/filepath' do
- expect_next_instance_of(::Gitlab::SecretDetection::Scan) do |instance|
+ expect_next_instance_of(::Gitlab::SecretDetection::Core::Scanner) do |instance|
expect(instance).to receive(:secrets_scan)
.with(
[new_payload],
@@ -839,27 +883,36 @@
let(:new_blob_reference2) { '5f571267577ed6e0b4b24fb87f7a8218d5912eb9' }
let(:new_payload) do
- { data: "SECRET=glpat-JUST20LETTERSANDNUMB", id: new_blob_reference, offset: 1 } # gitleaks:allow
+ ::Gitlab::SecretDetection::GRPC::ScanRequest::Payload.new(
+ data: "SECRET=glpat-JUST20LETTERSANDNUMB", # gitleaks:allow
+ id: new_blob_reference,
+ offset: 1
+ )
end
let(:new_payload2) do
- { data: "SECRET=glrt-JUST20LETTERSANDNUMB", id: new_blob_reference2, offset: 1 } # gitleaks:allow
+ ::Gitlab::SecretDetection::GRPC::ScanRequest::Payload.new(
+ data: "SECRET=glrt-JUST20LETTERSANDNUMB", # gitleaks:allow
+ id: new_blob_reference2,
+ offset: 1
+ )
end
let(:successful_with_multiple_files_findings_scan_response) do
- ::Gitlab::SecretDetection::Response.new(
- Gitlab::SecretDetection::Status::FOUND,
+ ::Gitlab::SecretDetection::Core::Response.new(
+ status: ::Gitlab::SecretDetection::Core::Status::FOUND,
+ results:
[
- Gitlab::SecretDetection::Finding.new(
+ ::Gitlab::SecretDetection::Core::Finding.new(
new_blob_reference,
- Gitlab::SecretDetection::Status::FOUND,
+ ::Gitlab::SecretDetection::Core::Status::FOUND,
1,
"gitlab_personal_access_token",
"GitLab Personal Access Token"
),
- Gitlab::SecretDetection::Finding.new(
+ ::Gitlab::SecretDetection::Core::Finding.new(
new_blob_reference2,
- Gitlab::SecretDetection::Status::FOUND,
+ ::Gitlab::SecretDetection::Core::Status::FOUND,
1,
"gitlab_runner_authentication_token",
"GitLab Runner Authentication Token"
@@ -869,7 +922,7 @@
end
it 'displays all findings with their corresponding commit sha/filepath' do
- expect_next_instance_of(::Gitlab::SecretDetection::Scan) do |instance|
+ expect_next_instance_of(::Gitlab::SecretDetection::Core::Scanner) do |instance|
expect(instance).to receive(:secrets_scan)
.with(
array_including(new_payload, new_payload2),
@@ -906,7 +959,7 @@
let(:category) { described_class.name }
before do
- allow_next_instance_of(::Gitlab::SecretDetection::Scan) do |instance|
+ allow_next_instance_of(::Gitlab::SecretDetection::Core::Scanner) do |instance|
allow(instance).to receive(:secrets_scan)
.with(
[new_payload],
@@ -953,8 +1006,8 @@
expect(hunk_header).to match(hunk_header_regex)
expect(offset).to eq(expected_offset)
- diff_blob = Gitlab::GitalyClient::DiffBlob.new(
- left_blob_id: Gitlab::Git::SHA1_BLANK_SHA,
+ diff_blob = ::Gitlab::GitalyClient::DiffBlob.new(
+ left_blob_id: ::Gitlab::Git::SHA1_BLANK_SHA,
right_blob_id: new_blob_reference,
patch: "#{hunk_header}\n+BASE_URL=https://foo.bar\n\\ No newline at end of file\n",
status: :STATUS_END_OF_PATCH,
@@ -1001,8 +1054,8 @@
it 'does not match invalid hunk header and skips parsing the diff' do
error_msg = "Could not process hunk header: #{hunk_header.strip}, skipped parsing diff: #{new_blob_reference}"
- diff_blob = Gitlab::GitalyClient::DiffBlob.new(
- left_blob_id: Gitlab::Git::SHA1_BLANK_SHA,
+ diff_blob = ::Gitlab::GitalyClient::DiffBlob.new(
+ left_blob_id: ::Gitlab::Git::SHA1_BLANK_SHA,
right_blob_id: new_blob_reference,
patch: "#{hunk_header}\n+BASE_URL=https://foo.bar\n\\ No newline at end of file\n",
status: :STATUS_END_OF_PATCH,
@@ -1041,8 +1094,8 @@
error_msg = "Could not process hunk header: #{invalid_header.strip}, skipped parsing diff: #{new_blob_reference}"
- diff_blob = Gitlab::GitalyClient::DiffBlob.new(
- left_blob_id: Gitlab::Git::SHA1_BLANK_SHA,
+ diff_blob = ::Gitlab::GitalyClient::DiffBlob.new(
+ left_blob_id: ::Gitlab::Git::SHA1_BLANK_SHA,
right_blob_id: new_blob_reference,
patch: patch,
status: :STATUS_END_OF_PATCH,
@@ -1074,20 +1127,24 @@
let(:new_blob_reference) { 'fe29d93da4843da433e62711ace82db601eb4f8f' }
let(:new_payload) do
- { data: "SECRET=glpat-JUST20LETTERSANDNUMB", # gitleaks:allow
- id: new_blob_reference, offset: 1 }
+ ::Gitlab::SecretDetection::GRPC::ScanRequest::Payload.new(
+ data: "SECRET=glpat-JUST20LETTERSANDNUMB", # gitleaks:allow
+ id: new_blob_reference,
+ offset: 1
+ )
end
# The new commit must have a secret, so create a commit with one.
let_it_be(:new_commit) { create_commit('.env' => 'SECRET=glpat-JUST20LETTERSANDNUMB') } # gitleaks:allow
let(:successful_scan_response) do
- ::Gitlab::SecretDetection::Response.new(
- Gitlab::SecretDetection::Status::FOUND,
+ ::Gitlab::SecretDetection::Core::Response.new(
+ status: ::Gitlab::SecretDetection::Core::Status::FOUND,
+ results:
[
- Gitlab::SecretDetection::Finding.new(
+ ::Gitlab::SecretDetection::Core::Finding.new(
new_blob_reference,
- Gitlab::SecretDetection::Status::FOUND,
+ ::Gitlab::SecretDetection::Core::Status::FOUND,
1,
"gitlab_personal_access_token",
"GitLab Personal Access Token"
@@ -1097,8 +1154,8 @@
end
let(:diff_blob) do
- Gitlab::GitalyClient::DiffBlob.new(
- left_blob_id: Gitlab::Git::SHA1_BLANK_SHA,
+ ::Gitlab::GitalyClient::DiffBlob.new(
+ left_blob_id: ::Gitlab::Git::SHA1_BLANK_SHA,
right_blob_id: new_blob_reference,
patch: "@@ -0,0 +1 @@\n+SECRET=glpat-JUST20LETTERSANDNUMB\n\\ No newline at end of file\n", # gitleaks:allow
status: :STATUS_END_OF_PATCH,
@@ -1122,7 +1179,7 @@
end
it 'scans diffs' do
- expect_next_instance_of(::Gitlab::SecretDetection::Scan) do |instance|
+ expect_next_instance_of(::Gitlab::SecretDetection::Core::Scanner) do |instance|
expect(instance).to receive(:secrets_scan)
.with(
[new_payload],
@@ -1153,8 +1210,16 @@
context 'when multiple hunks exist in a single diff patch leading to multiple payloads' do
let(:new_payloads) do
[
- { data: "SECRET=glpat-JUST20LETTERSANDNUMB", id: new_blob_reference, offset: 1 }, # gitleaks:allow
- { data: "TOKEN=glpat-JUST20LETTERSANDNUMB", id: new_blob_reference, offset: 11 } # gitleaks:allow
+ ::Gitlab::SecretDetection::GRPC::ScanRequest::Payload.new(
+ data: "SECRET=glpat-JUST20LETTERSANDNUMB", # gitleaks:allow
+ id: new_blob_reference,
+ offset: 1
+ ),
+ ::Gitlab::SecretDetection::GRPC::ScanRequest::Payload.new(
+ data: "TOKEN=glpat-JUST20LETTERSANDNUMB", # gitleaks:allow
+ id: new_blob_reference,
+ offset: 11
+ )
]
end
@@ -1174,19 +1239,20 @@
end
let(:successful_scan_response) do
- ::Gitlab::SecretDetection::Response.new(
- Gitlab::SecretDetection::Status::FOUND,
+ ::Gitlab::SecretDetection::Core::Response.new(
+ status: ::Gitlab::SecretDetection::Core::Status::FOUND,
+ results:
[
- Gitlab::SecretDetection::Finding.new(
+ ::Gitlab::SecretDetection::Core::Finding.new(
new_blob_reference,
- Gitlab::SecretDetection::Status::FOUND,
+ ::Gitlab::SecretDetection::Core::Status::FOUND,
1,
"gitlab_personal_access_token",
"GitLab Personal Access Token"
),
- Gitlab::SecretDetection::Finding.new(
+ ::Gitlab::SecretDetection::Core::Finding.new(
new_blob_reference,
- Gitlab::SecretDetection::Status::FOUND,
+ ::Gitlab::SecretDetection::Core::Status::FOUND,
11,
"gitlab_personal_access_token",
"GitLab Personal Access Token"
@@ -1197,8 +1263,8 @@
context 'when encountering a new hunk header' do
let(:diff_blob) do
- Gitlab::GitalyClient::DiffBlob.new(
- left_blob_id: Gitlab::Git::SHA1_BLANK_SHA,
+ ::Gitlab::GitalyClient::DiffBlob.new(
+ left_blob_id: ::Gitlab::Git::SHA1_BLANK_SHA,
right_blob_id: new_blob_reference,
patch: encounter_hunk_header_patch,
status: :STATUS_END_OF_PATCH,
@@ -1228,7 +1294,7 @@
.and_return([diff_blob])
end
- expect_next_instance_of(::Gitlab::SecretDetection::Scan) do |instance|
+ expect_next_instance_of(::Gitlab::SecretDetection::Core::Scanner) do |instance|
expect(instance).to receive(:secrets_scan)
.with(
new_payloads,
@@ -1258,8 +1324,8 @@
context 'when encountering a context line' do
let(:diff_blob) do
- Gitlab::GitalyClient::DiffBlob.new(
- left_blob_id: Gitlab::Git::SHA1_BLANK_SHA,
+ ::Gitlab::GitalyClient::DiffBlob.new(
+ left_blob_id: ::Gitlab::Git::SHA1_BLANK_SHA,
right_blob_id: new_blob_reference,
patch: encounter_context_line_patch,
status: :STATUS_END_OF_PATCH,
@@ -1289,7 +1355,7 @@
.and_return([diff_blob])
end
- expect_next_instance_of(::Gitlab::SecretDetection::Scan) do |instance|
+ expect_next_instance_of(::Gitlab::SecretDetection::Core::Scanner) do |instance|
expect(instance).to receive(:secrets_scan)
.with(
new_payloads,
@@ -1323,23 +1389,24 @@
include_context 'secrets check context'
let(:successful_scan_with_errors_response) do
- ::Gitlab::SecretDetection::Response.new(
- Gitlab::SecretDetection::Status::FOUND_WITH_ERRORS,
+ ::Gitlab::SecretDetection::Core::Response.new(
+ status: ::Gitlab::SecretDetection::Core::Status::FOUND_WITH_ERRORS,
+ results:
[
- Gitlab::SecretDetection::Finding.new(
+ ::Gitlab::SecretDetection::Core::Finding.new(
new_blob_reference,
- Gitlab::SecretDetection::Status::FOUND,
+ ::Gitlab::SecretDetection::Core::Status::FOUND,
1,
"gitlab_personal_access_token",
"GitLab Personal Access Token"
),
- Gitlab::SecretDetection::Finding.new(
+ ::Gitlab::SecretDetection::Core::Finding.new(
timed_out_blob_reference,
- Gitlab::SecretDetection::Status::PAYLOAD_TIMEOUT
+ ::Gitlab::SecretDetection::Core::Status::PAYLOAD_TIMEOUT
),
- Gitlab::SecretDetection::Finding.new(
+ ::Gitlab::SecretDetection::Core::Finding.new(
failed_to_scan_blob_reference,
- Gitlab::SecretDetection::Status::SCAN_ERROR
+ ::Gitlab::SecretDetection::Core::Status::SCAN_ERROR
)
]
)
@@ -1365,20 +1432,32 @@
let(:timed_out_blob_reference) { 'eaf3c09526f50b5e35a096ef70cca033f9974653' }
let(:failed_to_scan_blob_reference) { '4fbec77313fd240d00fc37e522d0274b8fb54bd1' }
- let(:new_blob) { have_attributes(class: Gitlab::Git::Blob, id: new_blob_reference, size: 33) }
- let(:timed_out_blob) { have_attributes(class: Gitlab::Git::Blob, id: timed_out_blob_reference, size: 32) }
- let(:failed_to_scan_blob) { have_attributes(class: Gitlab::Git::Blob, id: failed_to_scan_blob_reference, size: 32) }
+ let(:new_blob) { have_attributes(class: ::Gitlab::Git::Blob, id: new_blob_reference, size: 33) }
+ let(:timed_out_blob) { have_attributes(class: ::Gitlab::Git::Blob, id: timed_out_blob_reference, size: 32) }
+ let(:failed_to_scan_blob) { have_attributes(class: ::Gitlab::Git::Blob, id: failed_to_scan_blob_reference, size: 32) }
let(:new_payload) do
- { data: "SECRET=glpat-JUST20LETTERSANDNUMB", id: new_blob_reference, offset: 1 } # gitleaks:allow
+ ::Gitlab::SecretDetection::GRPC::ScanRequest::Payload.new(
+ data: "SECRET=glpat-JUST20LETTERSANDNUMB", # gitleaks:allow
+ id: new_blob_reference,
+ offset: 1
+ )
end
let(:timed_out_payload) do
- { data: "TOKEN=glpat-JUST20LETTERSANDNUMB", id: timed_out_blob_reference, offset: 1 } # gitleaks:allow
+ ::Gitlab::SecretDetection::GRPC::ScanRequest::Payload.new(
+ data: "TOKEN=glpat-JUST20LETTERSANDNUMB", # gitleaks:allow
+ id: timed_out_blob_reference,
+ offset: 1
+ )
end
let(:failed_to_scan_payload) do
- { data: "GLPAT=glpat-JUST20LETTERSANDNUMB", id: failed_to_scan_blob_reference, offset: 1 } # gitleaks:allow
+ ::Gitlab::SecretDetection::GRPC::ScanRequest::Payload.new(
+ data: "GLPAT=glpat-JUST20LETTERSANDNUMB", # gitleaks:allow
+ id: failed_to_scan_blob_reference,
+ offset: 1
+ )
end
# Used for the quarantine directory context below.
@@ -1402,7 +1481,7 @@
it 'lists all blobs of a repository' do
expect(repository).to receive(:list_all_blobs)
.with(
- bytes_limit: Gitlab::Checks::SecretsCheck::PAYLOAD_BYTES_LIMIT + 1,
+ bytes_limit: ::Gitlab::Checks::SecretsCheck::PAYLOAD_BYTES_LIMIT + 1,
dynamic_timeout: kind_of(Float),
ignore_alternate_object_directories: true
)
@@ -1410,7 +1489,7 @@
.and_return([existing_blob, new_blob, timed_out_blob, failed_to_scan_blob])
.and_call_original
- expect_next_instance_of(::Gitlab::SecretDetection::Scan) do |instance|
+ expect_next_instance_of(::Gitlab::SecretDetection::Core::Scanner) do |instance|
expect(instance).to receive(:secrets_scan)
.with(
array_including(new_payload, timed_out_payload, failed_to_scan_payload),
@@ -1437,7 +1516,7 @@
.and_return([new_blob, timed_out_blob, failed_to_scan_blob])
end
- expect_next_instance_of(::Gitlab::SecretDetection::Scan) do |instance|
+ expect_next_instance_of(::Gitlab::SecretDetection::Core::Scanner) do |instance|
expect(instance).to receive(:secrets_scan)
.with(
array_including(new_payload, timed_out_payload, failed_to_scan_payload),
@@ -1460,7 +1539,7 @@
expect(repository).to receive(:list_blobs)
.with(
['--not', '--all', '--not'] + changes.pluck(:newrev),
- bytes_limit: Gitlab::Checks::SecretsCheck::PAYLOAD_BYTES_LIMIT + 1,
+ bytes_limit: ::Gitlab::Checks::SecretsCheck::PAYLOAD_BYTES_LIMIT + 1,
with_paths: false,
dynamic_timeout: kind_of(Float)
)
@@ -1468,7 +1547,7 @@
.and_return([new_blob, timed_out_blob, failed_to_scan_blob])
.and_call_original
- expect_next_instance_of(::Gitlab::SecretDetection::Scan) do |instance|
+ expect_next_instance_of(::Gitlab::SecretDetection::Core::Scanner) do |instance|
expect(instance).to receive(:secrets_scan)
.with(
array_including(new_payload, timed_out_payload, failed_to_scan_payload),
@@ -1488,7 +1567,7 @@
end
it 'scans diffs' do
- expect_next_instance_of(::Gitlab::SecretDetection::Scan) do |instance|
+ expect_next_instance_of(::Gitlab::SecretDetection::Core::Scanner) do |instance|
expect(instance).to receive(:secrets_scan)
.with(
array_including(new_payload, timed_out_payload, failed_to_scan_payload),
@@ -1526,7 +1605,7 @@
end
it 'loads tree entries of the new commit' do
- expect_next_instance_of(::Gitlab::SecretDetection::Scan) do |instance|
+ expect_next_instance_of(::Gitlab::SecretDetection::Core::Scanner) do |instance|
expect(instance).to receive(:secrets_scan)
.with(
array_including(new_payload, timed_out_payload, failed_to_scan_payload),
@@ -1581,42 +1660,46 @@
let(:new_blob_reference) { '59ef300b246861163ee1e2ab4146e16144e4770f' }
let(:new_payload) do
- { data: "SECRET=glpat-JUST20LETTERSANDNUMB\nTOKEN=glpat-JUST20LETTERSANDNUMB", # gitleaks:allow
- id: new_blob_reference, offset: 1 }
+ ::Gitlab::SecretDetection::GRPC::ScanRequest::Payload.new(
+ data: "SECRET=glpat-JUST20LETTERSANDNUMB\nTOKEN=glpat-JUST20LETTERSANDNUMB", # gitleaks:allow
+ id: new_blob_reference,
+ offset: 1
+ )
end
let(:successful_scan_with_multiple_findings_and_errors_response) do
- ::Gitlab::SecretDetection::Response.new(
- Gitlab::SecretDetection::Status::FOUND_WITH_ERRORS,
+ ::Gitlab::SecretDetection::Core::Response.new(
+ status: ::Gitlab::SecretDetection::Core::Status::FOUND_WITH_ERRORS,
+ results:
[
- Gitlab::SecretDetection::Finding.new(
+ ::Gitlab::SecretDetection::Core::Finding.new(
new_blob_reference,
- Gitlab::SecretDetection::Status::FOUND,
+ ::Gitlab::SecretDetection::Core::Status::FOUND,
1,
"gitlab_personal_access_token",
"GitLab Personal Access Token"
),
- Gitlab::SecretDetection::Finding.new(
+ ::Gitlab::SecretDetection::Core::Finding.new(
new_blob_reference,
- Gitlab::SecretDetection::Status::FOUND,
+ ::Gitlab::SecretDetection::Core::Status::FOUND,
2,
"gitlab_personal_access_token",
"GitLab Personal Access Token"
),
- Gitlab::SecretDetection::Finding.new(
+ ::Gitlab::SecretDetection::Core::Finding.new(
timed_out_blob_reference,
- Gitlab::SecretDetection::Status::PAYLOAD_TIMEOUT
+ ::Gitlab::SecretDetection::Core::Status::PAYLOAD_TIMEOUT
),
- Gitlab::SecretDetection::Finding.new(
+ ::Gitlab::SecretDetection::Core::Finding.new(
failed_to_scan_blob_reference,
- Gitlab::SecretDetection::Status::SCAN_ERROR
+ ::Gitlab::SecretDetection::Core::Status::SCAN_ERROR
)
]
)
end
it 'displays all findings with their corresponding commit sha/filepath' do
- expect_next_instance_of(::Gitlab::SecretDetection::Scan) do |instance|
+ expect_next_instance_of(::Gitlab::SecretDetection::Core::Scanner) do |instance|
expect(instance).to receive(:secrets_scan)
.with(
array_including(new_payload, timed_out_payload, failed_to_scan_payload),
@@ -1658,12 +1741,12 @@
include_context 'secrets check context'
let(:scan_timed_out_scan_response) do
- ::Gitlab::SecretDetection::Response.new(Gitlab::SecretDetection::Status::SCAN_TIMEOUT)
+ ::Gitlab::SecretDetection::Core::Response.new(status: ::Gitlab::SecretDetection::Core::Status::SCAN_TIMEOUT)
end
it 'logs the error and passes the check' do
# Mock the response to return a scan timed out status.
- expect_next_instance_of(::Gitlab::SecretDetection::Scan) do |instance|
+ expect_next_instance_of(::Gitlab::SecretDetection::Core::Scanner) do |instance|
expect(instance).to receive(:secrets_scan)
.and_return(scan_timed_out_scan_response)
end
@@ -1682,22 +1765,29 @@
before do
# Intentionally set `RULESET_FILE_PATH` to an incorrect path to cause error.
- stub_const('::Gitlab::SecretDetection::Scan::RULESET_FILE_PATH', 'gitleaks.toml')
+ stub_const('::Gitlab::SecretDetection::Core::Ruleset::RULESET_FILE_PATH', 'gitleaks.toml')
end
it 'logs the error and passes the check' do
+ allow(TomlRB).to receive(:load_file).and_raise(
+ StandardError,
+ "No such file or directory @ rb_sysopen - gitleaks.toml"
+ )
+
+ error_msg = "No such file or directory @ rb_sysopen - gitleaks.toml"
+ error_string = "Failed to parse secret detection ruleset from 'gitleaks.toml' path: #{error_msg}"
+
# File parsing error is written to the logger.
expect(secret_detection_logger).to receive(:error)
.once
- .with(
- "Failed to parse secret detection ruleset from 'gitleaks.toml' path: " \
- "No such file or directory @ rb_sysopen - gitleaks.toml"
- )
+ .with(error_string)
+
+ msg = format(error_messages[:scan_initialization_error], { error_msg: error_msg })
# Error bubbles up from scan class and is handled in secrets check.
expect(secret_detection_logger).to receive(:error)
.once
- .with(message: error_messages[:scan_initialization_error])
+ .with(message: msg)
expect { subject.validate! }.not_to raise_error
end
@@ -1707,12 +1797,12 @@
include_context 'secrets check context'
let(:failed_with_invalid_input_response) do
- ::Gitlab::SecretDetection::Response.new(::Gitlab::SecretDetection::Status::INPUT_ERROR)
+ ::Gitlab::SecretDetection::Core::Response.new(status: ::Gitlab::SecretDetection::Core::Status::INPUT_ERROR)
end
it 'logs the error and passes the check' do
# Mock the response to return a scan invalid input status.
- expect_next_instance_of(::Gitlab::SecretDetection::Scan) do |instance|
+ expect_next_instance_of(::Gitlab::SecretDetection::Core::Scanner) do |instance|
expect(instance).to receive(:secrets_scan)
.and_return(failed_with_invalid_input_response)
end
@@ -1729,12 +1819,16 @@
RSpec.shared_examples 'scan skipped due to invalid status' do
include_context 'secrets check context'
- let(:invalid_scan_status_code) { -1 } # doesn't exist in ::Gitlab::SecretDetection::Status
- let(:invalid_scan_status_code_response) { ::Gitlab::SecretDetection::Response.new(invalid_scan_status_code) }
+ let(:invalid_scan_status_code) { -1 } # doesn't exist in ::Gitlab::SecretDetection::Core::Status
+ let(:invalid_scan_status_code_response) do
+ ::Gitlab::SecretDetection::Core::Response.new(
+ status: invalid_scan_status_code
+ )
+ end
it 'logs the error and passes the check' do
# Mock the response to return a scan invalid status.
- expect_next_instance_of(::Gitlab::SecretDetection::Scan) do |instance|
+ expect_next_instance_of(::Gitlab::SecretDetection::Core::Scanner) do |instance|
expect(instance).to receive(:secrets_scan)
.and_return(invalid_scan_status_code_response)
end
@@ -1798,13 +1892,13 @@
include_context 'secrets check context'
let(:changes_access) do
- Gitlab::Checks::ChangesAccess.new(
+ ::Gitlab::Checks::ChangesAccess.new(
changes,
project: project,
user_access: user_access,
protocol: protocol,
logger: logger,
- push_options: Gitlab::PushOptions.new(["secret_push_protection.skip_all"])
+ push_options: ::Gitlab::PushOptions.new(["secret_push_protection.skip_all"])
)
end
@@ -2113,3 +2207,115 @@
end
end
end
+
+RSpec.shared_examples 'detects secrets with special characters in diffs' do
+ include_context 'secrets check context'
+ include_context 'special characters table'
+
+ with_them do
+ let(:secret_with_special_char) { "SECRET=glpat-JUST20LETTERSANDNUMB #{special_character}" } # gitleaks:allow
+
+ let(:diff_blob) do
+ ::Gitlab::GitalyClient::DiffBlob.new(
+ left_blob_id: ::Gitlab::Git::SHA1_BLANK_SHA,
+ right_blob_id: new_blob_reference,
+ patch: "@@ -0,0 +1 @@\n+#{secret_with_special_char}\n\\ No newline at end of file\n",
+ status: :STATUS_END_OF_PATCH,
+ binary: false,
+ over_patch_bytes_limit: false
+ )
+ end
+
+ let(:new_payload) do
+ ::Gitlab::SecretDetection::GRPC::ScanRequest::Payload.new(
+ data: secret_with_special_char.force_encoding("UTF-8"),
+ id: new_blob_reference,
+ offset: 1
+ )
+ end
+
+ it "detects secret in diff containing #{params[:description]}" do
+ expect_next_instance_of(described_class) do |instance|
+ expect(instance).to receive(:get_diffs)
+ .once
+ .and_return([diff_blob])
+ end
+
+ expect_next_instance_of(::Gitlab::SecretDetection::Core::Scanner) do |instance|
+ expect(instance).to receive(:secrets_scan)
+ .with(
+ [new_payload],
+ timeout: kind_of(Float),
+ exclusions: kind_of(Hash)
+ )
+ .once
+ .and_call_original
+ end
+
+ expect(secret_detection_logger).to receive(:info)
+ .once
+ .with(message: log_messages[:found_secrets])
+
+ expect { subject.validate! }.to raise_error(::Gitlab::GitAccess::ForbiddenError)
+ end
+ end
+end
+
+RSpec.shared_examples 'detects secrets with special characters in full files' do
+ include_context 'secrets check context'
+ include_context 'special characters table'
+
+ before do
+ stub_feature_flags(spp_scan_diffs: false)
+ end
+
+ with_them do
+ let(:secret_with_special_char) { "SECRET=glpat-JUST20LETTERSANDNUMB #{special_character}" } # gitleaks:allow
+
+ let(:new_blob) do
+ instance_double(::Gitlab::Git::Blob,
+ id: new_blob_reference,
+ size: secret_with_special_char.bytesize,
+ binary: false,
+ data: secret_with_special_char
+ )
+ end
+
+ let(:new_payload) do
+ ::Gitlab::SecretDetection::GRPC::ScanRequest::Payload.new(
+ data: secret_with_special_char.dup.force_encoding("UTF-8"),
+ id: new_blob_reference,
+ offset: 1
+ )
+ end
+
+ let(:new_commit) do
+ create_commit('.env' => secret_with_special_char)
+ end
+
+ it "detects secret in file containing #{params[:description]}" do
+ expect_next_instance_of(::Gitlab::Checks::ChangedBlobs) do |instance|
+ expect(instance).to receive(:execute)
+ .once
+ .and_return([new_blob])
+ end
+
+ expect_next_instance_of(::Gitlab::SecretDetection::Core::Scanner) do |instance|
+ expect(instance).to receive(:secrets_scan)
+ .with(
+ [new_payload],
+ timeout: kind_of(Float),
+ exclusions: kind_of(Hash)
+ )
+ .once
+ .and_call_original
+ end
+
+ expect(secret_detection_logger).to receive(:info)
+ .once
+ .with(message: log_messages[:found_secrets])
+
+ expect { subject.validate! }.to raise_error(::Gitlab::GitAccess::ForbiddenError)
+ end
+ end
+end