Merge branch '427723-add-bin-saas_feature' into 'master'

Add bin/saas-feature helper script and document

See merge request https://gitlab.com/gitlab-org/gitlab/-/merge_requests/141701



Merged-by: Lin Jen-Shin <jen-shin@gitlab.com>
Approved-by: Lin Jen-Shin <jen-shin@gitlab.com>
Reviewed-by: Doug Stull <dstull@gitlab.com>
Reviewed-by: Lin Jen-Shin <jen-shin@gitlab.com>
Co-authored-by: Doug Stull <dstull@gitlab.com>

bin/saas-feature.rb

+#!/usr/bin/env ruby
+#
+# Generate a SaaS feature entry file in the correct location.
+#
+# Automatically stages the file and amends the previous commit if the `--amend`
+# argument is used.
+
+require 'fileutils'
+require 'httparty'
+require 'json'
+require 'optparse'
+require 'readline'
+require 'shellwords'
+require 'uri'
+require 'yaml'
+
+require_relative '../lib/gitlab/popen'
+
+module SaasFeatureHelpers
+  Abort = Class.new(StandardError)
+  Done = Class.new(StandardError)
+
+  def capture_stdout(cmd)
+    output = IO.popen(cmd, &:read)
+    fail_with "command failed: #{cmd.join(' ')}" unless $?.success?
+    output
+  end
+
+  def fail_with(message)
+    raise Abort, "\e[31merror\e[0m #{message}"
+  end
+end
+
+class SaasFeatureOptionParser
+  extend SaasFeatureHelpers
+
+  WWW_GITLAB_COM_SITE = 'https://about.gitlab.com'
+  WWW_GITLAB_COM_GROUPS_JSON = "#{WWW_GITLAB_COM_SITE}/groups.json".freeze
+  COPY_COMMANDS = [
+    'pbcopy', # macOS
+    'xclip -selection clipboard', # Linux
+    'xsel --clipboard --input', # Linux
+    'wl-copy' # Wayland
+  ].freeze
+  OPEN_COMMANDS = [
+    'open', # macOS
+    'xdg-open' # Linux
+  ].freeze
+
+  Options = Struct.new(
+    :name,
+    :group,
+    :milestone,
+    :amend,
+    :dry_run,
+    :force,
+    :introduced_by_url,
+    keyword_init: true
+  )
+
+  class << self
+    def parse(argv)
+      options = Options.new
+
+      parser = OptionParser.new do |opts|
+        opts.banner = "Usage: #{__FILE__} [options] <saas-feature>\n\n"
+
+        # Note: We do not provide a shorthand for this in order to match the `git
+        # commit` interface
+        opts.on('--amend', 'Amend the previous commit') do |value|
+          options.amend = value
+        end
+
+        opts.on('-f', '--force', 'Overwrite an existing entry') do |value|
+          options.force = value
+        end
+
+        opts.on('-m', '--introduced-by-url [string]', String, 'URL of merge request introducing the SaaS feature') do |value|
+          options.introduced_by_url = value
+        end
+
+        opts.on('-M', '--milestone [string]', String, 'Milestone in which the SaaS feature was introduced') do |value|
+          options.milestone = value
+        end
+
+        opts.on('-n', '--dry-run', "Don't actually write anything, just print") do |value|
+          options.dry_run = value
+        end
+
+        opts.on('-g', '--group [string]', String, 'The group introducing a SaaS feature, like: `group::project management`') do |value|
+          options.group = value if group_labels.include?(value)
+        end
+
+        opts.on('-h', '--help', 'Print help message') do
+          $stdout.puts opts
+          raise Done.new
+        end
+      end
+
+      parser.parse!(argv)
+
+      unless argv.one?
+        $stdout.puts parser.help
+        $stdout.puts
+        raise Abort, 'SaaS feature name is required'
+      end
+
+      # Name is a first name
+      options.name = argv.first.downcase.tr('-', '_')
+
+      options
+    end
+
+    def groups
+      @groups ||= fetch_json(WWW_GITLAB_COM_GROUPS_JSON)
+    end
+
+    def group_labels
+      @group_labels ||= groups.map { |_, group| group['label'] }.sort
+    end
+
+    def find_group_by_label(label)
+      groups.find { |_, group| group['label'] == label }[1]
+    end
+
+    def group_list
+      group_labels.map.with_index do |group_label, index|
+        "#{index + 1}. #{group_label}"
+      end
+    end
+
+    def fzf_available?
+      find_compatible_command(%w[fzf])
+    end
+
+    def prompt_readline(prompt:)
+      Readline.readline('?> ', false)&.strip
+    end
+
+    def prompt_fzf(list:, prompt:)
+      arr = list.join("\n")
+
+      selection = IO.popen(%W[fzf --tac --prompt #{prompt}], "r+") do |pipe|
+        pipe.puts(arr)
+        pipe.close_write
+        pipe.readlines
+      end.join.strip
+
+      selection[/(\d+)\./, 1]
+    end
+
+    def print_list(list)
+      return if list.empty?
+
+      $stdout.puts list.join("\n")
+    end
+
+    def print_prompt(prompt)
+      $stdout.puts
+      $stdout.puts ">> #{prompt}:"
+      $stdout.puts
+    end
+
+    def prompt_list(prompt:, list: nil)
+      if fzf_available?
+        prompt_fzf(list: list, prompt: prompt)
+      else
+        prompt_readline(prompt: prompt)
+      end
+    end
+
+    def fetch_json(json_url)
+      json = with_retries { HTTParty.get(json_url, format: :plain) }
+      JSON.parse(json)
+    end
+
+    def with_retries(attempts: 3)
+      yield
+    rescue Errno::ECONNRESET, OpenSSL::SSL::SSLError, Net::OpenTimeout
+      retry if (attempts -= 1).positive?
+      raise
+    end
+
+    def read_group
+      prompt = 'Specify the group label to which the SaaS feature belongs, from the following list'
+
+      unless fzf_available?
+        print_prompt(prompt)
+        print_list(group_list)
+      end
+
+      loop do
+        group = prompt_list(prompt: prompt, list: group_list)
+        group = group_labels[group.to_i - 1] unless group.to_i.zero?
+
+        if group_labels.include?(group)
+          $stdout.puts "You picked the group '#{group}'"
+          return group
+        else
+          $stderr.puts "The group label isn't in the above labels list"
+        end
+
+      end
+    end
+
+    def read_introduced_by_url
+      read_url('URL of the MR introducing the SaaS feature (enter to skip and let Danger provide a suggestion directly in the MR):')
+    end
+
+    def read_milestone
+      milestone = File.read('VERSION')
+      milestone.gsub(/^(\d+\.\d+).*$/, '\1').chomp
+    end
+
+    def read_url(prompt)
+      $stdout.puts
+      $stdout.puts ">> #{prompt}"
+
+      loop do
+        url = Readline.readline('?> ', false)&.strip
+        url = nil if url.empty?
+        return url if url.nil? || valid_url?(url)
+      end
+    end
+
+    def valid_url?(url)
+      unless url.start_with?('https://')
+        $stderr.puts 'URL needs to start with https://'
+        return false
+      end
+
+      response = HTTParty.head(url)
+
+      return true if response.success?
+
+      $stderr.puts "URL '#{url}' isn't valid!"
+    end
+
+    def open_url!(url)
+      _, open_url_status = Gitlab::Popen.popen([open_command, url])
+
+      open_url_status
+    end
+
+    def copy_to_clipboard!(text)
+      IO.popen(copy_to_clipboard_command.shellsplit, 'w') do |pipe|
+        pipe.print(text)
+      end
+    end
+
+    def copy_to_clipboard_command
+      find_compatible_command(COPY_COMMANDS)
+    end
+
+    def open_command
+      find_compatible_command(OPEN_COMMANDS)
+    end
+
+    def find_compatible_command(commands)
+      commands.find do |command|
+        Gitlab::Popen.popen(%W[which #{command.split(' ')[0]}])[1] == 0
+      end
+    end
+  end
+end
+
+class SaasFeatureCreator
+  include SaasFeatureHelpers
+
+  attr_reader :options
+
+  def initialize(options)
+    @options = options
+  end
+
+  def execute
+    assert_feature_branch!
+    assert_name!
+    assert_existing_saas_feature!
+
+    options.group ||= SaasFeatureOptionParser.read_group
+    options.introduced_by_url ||= SaasFeatureOptionParser.read_introduced_by_url
+    options.milestone ||= SaasFeatureOptionParser.read_milestone
+
+    $stdout.puts "\e[32mcreate\e[0m #{file_path}"
+    $stdout.puts contents
+
+    unless options.dry_run
+      write
+      amend_commit if options.amend
+    end
+
+    if editor
+      system(editor, file_path)
+    end
+  end
+
+  private
+
+  def contents
+    config_hash.to_yaml
+  end
+
+  def config_hash
+    {
+      'name'              => options.name,
+      'introduced_by_url' => options.introduced_by_url,
+      'milestone'         => options.milestone,
+      'group'             => options.group
+    }
+  end
+
+  def write
+    FileUtils.mkdir_p(File.dirname(file_path))
+    File.write(file_path, contents)
+  end
+
+  def editor
+    ENV['EDITOR']
+  end
+
+  def amend_commit
+    fail_with 'git add failed' unless system(*%W[git add #{file_path}])
+
+    system('git commit --amend')
+  end
+
+  def assert_feature_branch!
+    return unless branch_name == 'master'
+
+    fail_with 'Create a branch first!'
+  end
+
+  def assert_existing_saas_feature!
+    existing_path = all_saas_feature_names[options.name]
+    return unless existing_path
+    return if options.force
+
+    fail_with "#{existing_path} already exists! Use `--force` to overwrite."
+  end
+
+  def assert_name!
+    return if options.name.match(/\A[a-z0-9_-]+\Z/)
+
+    fail_with 'Provide a name for the SaaS feature that is [a-z0-9_-]'
+  end
+
+  def file_path
+    saas_features_path.sub('*.yml', options.name + '.yml')
+  end
+
+  def all_saas_feature_names
+    # check flatten needs
+    @all_saas_feature_names ||=
+      Dir.glob(saas_features_path).map do |path|
+        [File.basename(path, '.yml'), path]
+      end.to_h
+  end
+
+  def saas_features_path
+    File.join('ee', 'config', 'saas_features', '*.yml')
+  end
+
+  def branch_name
+    @branch_name ||= capture_stdout(%w[git symbolic-ref --short HEAD]).strip
+  end
+end
+
+if $0 == __FILE__
+  begin
+    options = SaasFeatureOptionParser.parse(ARGV)
+    SaasFeatureCreator.new(options).execute
+  rescue SaasFeatureHelpers::Abort => ex
+    $stderr.puts ex.message
+    exit 1
+  rescue SaasFeatureHelpers::Done
+    exit
+  end
+end
+
+# vim: ft=ruby

doc/development/ee_features.md

@@ -63,6 +63,29 @@ Each SaaS feature is defined in a separate YAML file consisting of a number of f
 | `milestone`         | no       | Milestone in which the SaaS feature was created.                                                             |
 | `group`             | no       | The [group](https://about.gitlab.com/handbook/product/categories/#devops-stages) that owns the feature flag. |
 
+#### Create a new SaaS feature file definition
+
+The GitLab codebase provides [`bin/saas-feature.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/bin/saas-feature.rb),
+a dedicated tool to create new SaaS feature definitions.
+The tool asks various questions about the new SaaS feature, then creates
+a YAML definition in `ee/config/saas_features`.
+
+Only SaaS features that have a YAML definition file can be used when running the development or testing environments.
+
+```shell
+❯ bin/saas-feature my_saas_feature
+You picked the group 'group::acquisition'
+
+>> URL of the MR introducing the SaaS feature (enter to skip and let Danger provide a suggestion directly in the MR):
+?> https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38602
+create ee/config/saas_features/my_saas_feature.yml
+---
+name: my_saas_feature
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38602
+milestone: '16.8'
+group: group::acquisition
+```
+
 ### Opting out of a SaaS-only feature on another SaaS instance (JiHu)
 
 Prepend the `ee/lib/ee/gitlab/saas.rb` module and override the `Gitlab::Saas.feature_available?` method.

spec/bin/saas_feature_spec.rb

+# frozen_string_literal: true
+
+require 'fast_spec_helper'
+require 'rspec-parameterized'
+
+require_relative '../../bin/saas-feature'
+
+RSpec.describe 'bin/saas-feature', feature_category: :feature_flags do
+  using RSpec::Parameterized::TableSyntax
+
+  let(:groups) { { geo: { label: 'group::geo' } } }
+
+  before do
+    allow(HTTParty)
+      .to receive(:get)
+        .with(SaasFeatureOptionParser::WWW_GITLAB_COM_GROUPS_JSON, format: :plain)
+        .and_return(groups.to_json)
+  end
+
+  describe SaasFeatureCreator do
+    let(:argv) { %w[saas-feature-name -g group::geo -m http://url -M 16.6] }
+    let(:options) { SaasFeatureOptionParser.parse(argv) }
+    let(:creator) { described_class.new(options) }
+    let(:existing_saas_features) do
+      { 'existing_saas_feature' => File.join('ee', 'config', 'saas_features', 'existing_saas_feature.yml') }
+    end
+
+    before do
+      allow(creator).to receive(:all_saas_feature_names) { existing_saas_features }
+      allow(creator).to receive(:branch_name).and_return('feature-branch')
+      allow(creator).to receive(:editor).and_return(nil)
+
+      # ignore writes
+      allow(File).to receive(:write).and_return(true)
+
+      # ignore stdin
+      allow(Readline).to receive(:readline).and_raise('EOF')
+    end
+
+    subject(:execute) { creator.execute }
+
+    it 'properly creates a SaaS feature' do
+      expect(File).to receive(:write).with(
+        File.join('ee', 'config', 'saas_features', 'saas_feature_name.yml'),
+        anything)
+
+      expect { execute }.to output(/name: saas_feature_name/).to_stdout
+    end
+
+    context 'when running on master' do
+      it 'requires feature branch' do
+        expect(creator).to receive(:branch_name).and_return('master')
+
+        expect { execute }.to raise_error(SaasFeatureHelpers::Abort, /Create a branch first/)
+      end
+    end
+
+    context 'with SaaS feature name validation' do
+      where(:argv, :ex) do
+        %w[.invalid.saas.feature] | /Provide a name for the SaaS feature that is/
+        %w[existing-saas-feature] | /already exists!/
+      end
+
+      with_them do
+        it do
+          expect { execute }.to raise_error(ex)
+        end
+      end
+    end
+  end
+
+  describe SaasFeatureOptionParser do
+    describe '.parse' do
+      where(:param, :argv, :result) do
+        :name              | %w[foo]                                 | 'foo'
+        :amend             | %w[foo --amend]                         | true
+        :force             | %w[foo -f]                              | true
+        :force             | %w[foo --force]                         | true
+        :introduced_by_url | %w[foo -m https://url]                  | 'https://url'
+        :introduced_by_url | %w[foo --introduced-by-url https://url] | 'https://url'
+        :dry_run           | %w[foo -n]                              | true
+        :dry_run           | %w[foo --dry-run]                       | true
+        :group             | %w[foo -g group::geo]                   | 'group::geo'
+        :group             | %w[foo --group group::geo]              | 'group::geo'
+        :group             | %w[foo -g invalid]                      | nil
+        :group             | %w[foo --group invalid]                 | nil
+      end
+
+      with_them do
+        it do
+          options = described_class.parse(Array(argv))
+
+          expect(options.public_send(param)).to eq(result)
+        end
+      end
+
+      it 'missing SaaS feature name' do
+        expect do
+          expect { described_class.parse(%w[--amend]) }.to output(/SaaS feature name is required/).to_stdout
+        end.to raise_error(SaasFeatureHelpers::Abort)
+      end
+
+      it 'parses -h' do
+        expect do
+          expect { described_class.parse(%w[foo -h]) }.to output(/Usage:/).to_stdout
+        end.to raise_error(SaasFeatureHelpers::Done)
+      end
+    end
+
+    describe '.read_group' do
+      before do
+        allow(described_class).to receive(:fzf_available?).and_return(false)
+      end
+
+      context 'when valid group is given' do
+        let(:group) { 'group::geo' }
+
+        it 'reads group from stdin' do
+          expect(Readline).to receive(:readline).and_return(group)
+          expect do
+            expect(described_class.read_group).to eq('group::geo')
+          end.to output(/Specify the group label to which the SaaS feature belongs, from the following list/).to_stdout
+        end
+      end
+
+      context 'when valid index is given' do
+        it 'picks the group successfully' do
+          expect(Readline).to receive(:readline).and_return('1')
+          expect do
+            expect(described_class.read_group).to eq('group::geo')
+          end.to output(/Specify the group label to which the SaaS feature belongs, from the following list/).to_stdout
+        end
+      end
+
+      context 'with invalid group given' do
+        let(:type) { 'invalid' }
+
+        it 'shows error message and retries' do
+          expect(Readline).to receive(:readline).and_return(type)
+          expect(Readline).to receive(:readline).and_raise('EOF')
+
+          expect do
+            expect { described_class.read_group }.to raise_error(/EOF/)
+          end.to output(/Specify the group label to which the SaaS feature belongs, from the following list/).to_stdout
+            .and output(/The group label isn't in the above labels list/).to_stderr
+        end
+      end
+
+      context 'when invalid index is given' do
+        it 'shows error message and retries' do
+          expect(Readline).to receive(:readline).and_return('12')
+          expect(Readline).to receive(:readline).and_raise('EOF')
+
+          expect do
+            expect { described_class.read_group }.to raise_error(/EOF/)
+          end.to output(/Specify the group label to which the SaaS feature belongs, from the following list/).to_stdout
+            .and output(/The group label isn't in the above labels list/).to_stderr
+        end
+      end
+    end
+
+    describe '.read_introduced_by_url' do
+      context 'with valid URL given' do
+        let(:url) { 'https://merge-request' }
+
+        it 'reads URL from stdin' do
+          expect(Readline).to receive(:readline).and_return(url)
+          expect(HTTParty).to receive(:head).with(url).and_return(instance_double(HTTParty::Response, success?: true))
+
+          expect do
+            expect(described_class.read_introduced_by_url).to eq('https://merge-request')
+          end.to output(/URL of the MR introducing the SaaS feature/).to_stdout
+        end
+      end
+
+      context 'with invalid URL given' do
+        let(:url) { 'https://invalid' }
+
+        it 'shows error message and retries' do
+          expect(Readline).to receive(:readline).and_return(url)
+          expect(HTTParty).to receive(:head).with(url).and_return(instance_double(HTTParty::Response, success?: false))
+          expect(Readline).to receive(:readline).and_raise('EOF')
+
+          expect do
+            expect { described_class.read_introduced_by_url }.to raise_error(/EOF/)
+          end.to output(/URL of the MR introducing the SaaS feature/).to_stdout
+                                                                     .and output(/URL '#{url}' isn't valid/).to_stderr
+        end
+      end
+
+      context 'with empty URL given' do
+        let(:url) { '' }
+
+        it 'skips entry' do
+          expect(Readline).to receive(:readline).and_return(url)
+
+          expect do
+            expect(described_class.read_introduced_by_url).to be_nil
+          end.to output(/URL of the MR introducing the SaaS feature/).to_stdout
+        end
+      end
+
+      context 'with a non-URL given' do
+        let(:url) { 'malformed' }
+
+        it 'shows error message and retries' do
+          expect(Readline).to receive(:readline).and_return(url)
+          expect(Readline).to receive(:readline).and_raise('EOF')
+
+          expect do
+            expect { described_class.read_introduced_by_url }.to raise_error(/EOF/)
+          end.to output(/URL of the MR introducing the SaaS feature/).to_stdout
+                                                                     .and output(/URL needs to start with/).to_stderr
+        end
+      end
+    end
+  end
+end