Merge branch 'sh-add-google-cdn' into 'master'

Add support for Google CDN

See merge request gitlab-org/gitlab!96336

app/uploaders/object_storage/cdn.rb

+# rubocop:disable Naming/FileName
+# frozen_string_literal: true
+
+require_relative 'cdn/google_cdn'
+
+module ObjectStorage
+  module CDN
+    module Concern
+      extend ActiveSupport::Concern
+
+      include Gitlab::Utils::StrongMemoize
+
+      def use_cdn?(request_ip)
+        return false unless cdn_options.is_a?(Hash) && cdn_options['provider']
+        return false unless cdn_provider
+
+        cdn_provider.use_cdn?(request_ip)
+      end
+
+      def cdn_signed_url
+        cdn_provider&.signed_url(path)
+      end
+
+      private
+
+      def cdn_provider
+        strong_memoize(:cdn_provider) do
+          provider = cdn_options['provider']&.downcase
+
+          next unless provider
+          next GoogleCDN.new(cdn_options) if provider == 'google'
+
+          raise "Unknown CDN provider: #{provider}"
+        end
+      end
+
+      def cdn_options
+        return {} unless options.object_store.key?('cdn')
+
+        options.object_store.cdn
+      end
+    end
+  end
+end
+
+# rubocop:enable Naming/FileName

app/uploaders/object_storage/cdn/google_cdn.rb

+# rubocop:disable Naming/FileName
+# frozen_string_literal: true
+
+module ObjectStorage
+  module CDN
+    class GoogleCDN
+      include Gitlab::Utils::StrongMemoize
+
+      IpListNotRetrievedError = Class.new(StandardError)
+
+      GOOGLE_CDN_LIST_KEY = 'google_cdn_ip_list'
+      GOOGLE_IP_RANGES_URL = 'https://www.gstatic.com/ipranges/cloud.json'
+      EXPECTED_CONTENT_TYPE = 'application/json'
+      RESPONSE_BODY_LIMIT = 1.megabyte
+      CACHE_EXPIRATION_TIME = 1.day
+
+      attr_reader :options
+
+      def initialize(options)
+        @options = HashWithIndifferentAccess.new(options.to_h)
+      end
+
+      def use_cdn?(request_ip)
+        return false unless config_valid?
+
+        ip = IPAddr.new(request_ip)
+
+        return false if ip.private?
+        return false unless google_ip_ranges.present?
+
+        !google_ip?(request_ip)
+      end
+
+      def signed_url(path, expiry: 10.minutes)
+        expiration = (Time.current + expiry).utc.to_i
+
+        uri = Addressable::URI.parse(cdn_url)
+        uri.path = path
+        uri.query = "Expires=#{expiration}&KeyName=#{key_name}"
+
+        signature = OpenSSL::HMAC.digest('SHA1', decoded_key, uri.to_s)
+        encoded_signature = Base64.urlsafe_encode64(signature)
+
+        uri.query += "&Signature=#{encoded_signature}"
+        uri.to_s
+      end
+
+      private
+
+      def config_valid?
+        [key_name, decoded_key, cdn_url].all?(&:present?)
+      end
+
+      def key_name
+        strong_memoize(:key_name) do
+          options['key_name']
+        end
+      end
+
+      def decoded_key
+        strong_memoize(:decoded_key) do
+          Base64.urlsafe_decode64(options['key']) if options['key']
+        rescue ArgumentError
+          Gitlab::ErrorTracking.log_exception(ArgumentError.new("Google CDN key is not base64-encoded"))
+          nil
+        end
+      end
+
+      def cdn_url
+        strong_memoize(:cdn_url) do
+          options['url']
+        end
+      end
+
+      def google_ip?(request_ip)
+        google_ip_ranges.any? { |range| range.include?(request_ip) }
+      end
+
+      def google_ip_ranges
+        strong_memoize(:google_ip_ranges) do
+          cache_value(GOOGLE_CDN_LIST_KEY) { fetch_google_ip_list }
+        end
+      rescue IpListNotRetrievedError => err
+        Gitlab::ErrorTracking.log_exception(err)
+        []
+      end
+
+      def cache_value(key, expires_in: CACHE_EXPIRATION_TIME, &block)
+        l1_cache.fetch(key, expires_in: expires_in) do
+          l2_cache.fetch(key, expires_in: expires_in) { yield }
+        end
+      end
+
+      def l1_cache
+        Gitlab::ProcessMemoryCache.cache_backend
+      end
+
+      def l2_cache
+        Rails.cache
+      end
+
+      def fetch_google_ip_list
+        response = Gitlab::HTTP.get(GOOGLE_IP_RANGES_URL)
+
+        raise IpListNotRetrievedError, "response was #{response.code}" unless response.code == 200
+
+        if response.body&.bytesize.to_i > RESPONSE_BODY_LIMIT
+          raise IpListNotRetrievedError, "response was too large: #{response.body.bytesize}"
+        end
+
+        parsed_response = response.parsed_response
+
+        unless response.content_type == EXPECTED_CONTENT_TYPE && parsed_response.is_a?(Hash)
+          raise IpListNotRetrievedError, "response was not JSON"
+        end
+
+        parse_google_prefixes(parsed_response)
+      end
+
+      def parse_google_prefixes(parsed_response)
+        prefixes = parsed_response['prefixes']
+
+        raise IpListNotRetrievedError, "JSON was type #{prefixes.class}, expected Array" unless prefixes.is_a?(Array)
+
+        ranges = prefixes.map do |prefix|
+          ip_range = prefix['ipv4Prefix'] || prefix['ipv6Prefix']
+
+          next unless ip_range
+
+          IPAddr.new(ip_range)
+        end.compact
+
+        raise IpListNotRetrievedError, "#{GOOGLE_IP_RANGES_URL} did not return any IP ranges" if ranges.empty?
+
+        ranges
+      end
+    end
+  end
+end
+
+# rubocop:enable Naming/FileName

config/initializers_before_autoloader/000_inflections.rb

@@ -37,6 +37,7 @@
     vulnerabilities_feedback
     vulnerability_feedback
   )
+  inflect.acronym 'CDN'
   inflect.acronym 'EE'
   inflect.acronym 'JH'
   inflect.acronym 'CSP'

spec/fixtures/cdn/google_cloud.json

+{
+  "syncToken": "1661533328840",
+  "creationTime": "2022-08-26T10:02:08.840384",
+  "prefixes": [{
+    "ipv4Prefix": "34.80.0.0/15",
+    "service": "Google Cloud",
+    "scope": "asia-east1"
+  }, {
+    "ipv4Prefix": "34.137.0.0/16",
+    "service": "Google Cloud",
+    "scope": "asia-east1"
+  }, {
+    "ipv6Prefix": "2600:1900:4180::/44",
+    "service": "Google Cloud",
+    "scope": "us-west4"
+  }]
+}

spec/uploaders/object_storage/cdn/google_cdn_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe ObjectStorage::CDN::GoogleCDN, :use_clean_rails_memory_store_caching do
+  include StubRequests
+
+  let(:key) { SecureRandom.hex }
+  let(:key_name) { 'test-key' }
+  let(:options) { { url: 'https://cdn.gitlab.example.com', key_name: key_name, key: Base64.urlsafe_encode64(key) } }
+  let(:google_cloud_ips) { File.read(Rails.root.join('spec/fixtures/cdn/google_cloud.json')) }
+  let(:headers) { { 'Content-Type' => 'application/json' } }
+  let(:public_ip) { '18.245.0.42' }
+
+  subject { described_class.new(options) }
+
+  before do
+    WebMock.stub_request(:get, described_class::GOOGLE_IP_RANGES_URL)
+      .to_return(status: 200, body: google_cloud_ips, headers: headers)
+  end
+
+  describe '#use_cdn?' do
+    using RSpec::Parameterized::TableSyntax
+
+    where(:ip_address, :expected) do
+      '34.80.0.1'                               | false
+      '18.245.0.42'                             | true
+      '2500:1900:4180:0000:0000:0000:0000:0000' | true
+      '2600:1900:4180:0000:0000:0000:0000:0000' | false
+      '10.10.1.5'                               | false
+      'fc00:0000:0000:0000:0000:0000:0000:0000' | false
+    end
+
+    with_them do
+      it { expect(subject.use_cdn?(ip_address)).to eq(expected) }
+    end
+
+    it 'caches the value' do
+      expect(subject.use_cdn?(public_ip)).to be true
+      expect(Rails.cache.fetch(described_class::GOOGLE_CDN_LIST_KEY)).to be_present
+      expect(Gitlab::ProcessMemoryCache.cache_backend.fetch(described_class::GOOGLE_CDN_LIST_KEY)).to be_present
+    end
+
+    context 'when the key name is missing' do
+      let(:options) { { url: 'https://cdn.gitlab.example.com', key: Base64.urlsafe_encode64(SecureRandom.hex) } }
+
+      it 'returns false' do
+        expect(subject.use_cdn?(public_ip)).to be false
+      end
+    end
+
+    context 'when the key is missing' do
+      let(:options) { { url: 'https://invalid.example.com' } }
+
+      it 'returns false' do
+        expect(subject.use_cdn?(public_ip)).to be false
+      end
+    end
+
+    context 'when the key is invalid' do
+      let(:options) { { key_name: key_name, key: '\0x1' } }
+
+      it 'returns false' do
+        expect(Gitlab::ErrorTracking).to receive(:log_exception).and_call_original
+        expect(subject.use_cdn?(public_ip)).to be false
+      end
+    end
+
+    context 'when the URL is missing' do
+      let(:options) { { key: Base64.urlsafe_encode64(SecureRandom.hex) } }
+
+      it 'returns false' do
+        expect(subject.use_cdn?(public_ip)).to be false
+      end
+    end
+
+    shared_examples 'IP range retrieval failure' do
+      it 'does not cache the result and logs an error' do
+        expect(Gitlab::ErrorTracking).to receive(:log_exception).and_call_original
+        expect(subject.use_cdn?(public_ip)).to be false
+        expect(Rails.cache.fetch(described_class::GOOGLE_CDN_LIST_KEY)).to be_nil
+        expect(Gitlab::ProcessMemoryCache.cache_backend.fetch(described_class::GOOGLE_CDN_LIST_KEY)).to be_nil
+      end
+    end
+
+    context 'when the URL returns a 404' do
+      before do
+        WebMock.stub_request(:get, described_class::GOOGLE_IP_RANGES_URL).to_return(status: 404)
+      end
+
+      it_behaves_like 'IP range retrieval failure'
+    end
+
+    context 'when the URL returns too large of a payload' do
+      before do
+        stub_const("#{described_class}::RESPONSE_BODY_LIMIT", 300)
+      end
+
+      it_behaves_like 'IP range retrieval failure'
+    end
+
+    context 'when the URL returns HTML' do
+      let(:headers) { { 'Content-Type' => 'text/html' } }
+
+      it_behaves_like 'IP range retrieval failure'
+    end
+
+    context 'when the URL returns empty results' do
+      let(:google_cloud_ips) { '{}' }
+
+      it_behaves_like 'IP range retrieval failure'
+    end
+  end
+
+  describe '#signed_url' do
+    let(:path) { '/path/to/file.txt' }
+
+    it 'returns a valid signed URL' do
+      url = subject.signed_url(path)
+
+      expect(url).to start_with("#{options[:url]}#{path}")
+
+      uri = Addressable::URI.parse(url)
+      parsed_query = Rack::Utils.parse_nested_query(uri.query)
+      signature = parsed_query.delete('Signature')
+
+      signed_url = "#{options[:url]}#{path}?Expires=#{parsed_query['Expires']}&KeyName=#{key_name}"
+      computed_signature = OpenSSL::HMAC.digest('SHA1', key, signed_url)
+
+      aggregate_failures do
+        expect(parsed_query['Expires'].to_i).to be > 0
+        expect(parsed_query['KeyName']).to eq(key_name)
+        expect(signature).to eq(Base64.urlsafe_encode64(computed_signature))
+      end
+    end
+  end
+end

spec/uploaders/object_storage/cdn_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe ObjectStorage::CDN do
+  let(:cdn_options) do
+    {
+      'object_store' => {
+        'cdn' => {
+          'provider' => 'google',
+          'url' => 'https://gitlab.example.com',
+          'key_name' => 'test-key',
+          'key' => '12345'
+        }
+      }
+    }.freeze
+  end
+
+  let(:uploader_class) do
+    Class.new(GitlabUploader) do
+      include ObjectStorage::Concern
+      include ObjectStorage::CDN::Concern
+
+      private
+
+      # user/:id
+      def dynamic_segment
+        File.join(model.class.underscore, model.id.to_s)
+      end
+    end
+  end
+
+  let(:object) { build_stubbed(:user) }
+
+  subject { uploader_class.new(object, :file) }
+
+  context 'with CDN config' do
+    before do
+      uploader_class.options = Settingslogic.new(Gitlab.config.uploads.deep_merge(cdn_options))
+    end
+
+    describe '#use_cdn?' do
+      it 'returns true' do
+        expect_next_instance_of(ObjectStorage::CDN::GoogleCDN) do |cdn|
+          expect(cdn).to receive(:use_cdn?).and_return(true)
+        end
+
+        expect(subject.use_cdn?('18.245.0.1')).to be true
+      end
+    end
+
+    describe '#cdn_signed_url' do
+      it 'returns a URL' do
+        expect_next_instance_of(ObjectStorage::CDN::GoogleCDN) do |cdn|
+          expect(cdn).to receive(:signed_url).and_return("https://cdn.example.com/path")
+        end
+
+        expect(subject.cdn_signed_url).to eq("https://cdn.example.com/path")
+      end
+    end
+  end
+
+  context 'without CDN config' do
+    before do
+      uploader_class.options = Gitlab.config.uploads
+    end
+
+    describe '#use_cdn?' do
+      it 'returns false' do
+        expect(subject.use_cdn?('18.245.0.1')).to be false
+      end
+    end
+  end
+
+  context 'with an unknown CDN provider' do
+    before do
+      cdn_options['object_store']['cdn']['provider'] = 'amazon'
+      uploader_class.options = Settingslogic.new(Gitlab.config.uploads.deep_merge(cdn_options))
+    end
+
+    it 'raises an error' do
+      expect { subject.use_cdn?('18.245.0.1') }.to raise_error("Unknown CDN provider: amazon")
+    end
+  end
+end