Remove deprecated /code_suggestions/tokens endpoint

* In an earlier iteration of code suggestions, this endpoint was used
  to generate a JWT for the IDE, which then made calls directly to the
  AI Gateway.
* This endpoint already returns a 404 for gitlab.com, but we kept it
  around for older self-managed GitLab instances that were using code
  suggestions.
* As of Feb 15th, all Code Suggestions users must be on 16.7 or newer if
  they are self managed so we can safely get rid of this endpoint.
* https://gitlab.com/gitlab-org/editor-extensions/gitlab-lsp/-/issues/137

Changelog: removed
EE: true

config/events/20230605152832_API__Ml__AiAssist_authenticate.yml

----
-description: An event triggered when a user is authorized to use Code Suggestions
-category: API::Ml::AiAssist
-action: authenticate
-label_description: code_suggestions
-property_description:
-value_description:
-extra_properties:
-identifiers:
-- user
-product_section: dev
-product_stage: modelops
-product_group: ai_assisted
-milestone: "16.1"
-introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/122774
-distributions:
-- ee
-tiers:
-- free
-- premium
-- ultimate
-

config/events/code_suggestions_authenticate.yml

----
-description: Tracks the number of authentication requests to the `POST /code_suggestions/tokens` API endpoint.
-category: InternalEventTracking
-action: code_suggestions_authenticate
-label_description:
-property_description:
-value_description:
-extra_properties:
-identifiers:
-- user
-product_section: dev
-product_stage: create
-product_group: code_creation
-milestone: "16.7"
-introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/138848
-distributions:
-- ce
-- ee
-tiers:
-- free
-- premium
-- ultimate

config/metrics/counts_28d/count_total_code_suggestions_authenticate_28d.yml

----
-key_path: count_total_code_suggestions_authenticate_28d
-description: Tracks the number of authentication requests to the `POST /code_suggestions/tokens` API endpoint in the last 28 days.
-product_section: dev
-product_stage: create
-product_group: code_creation
-performance_indicator_type: []
-value_type: number
-status: active
-milestone: "16.7"
-introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/138848
-time_frame: 28d
-data_source: internal_events
-data_category: optional
-distribution:
-- ce
-- ee
-tier:
-- free
-- premium
-- ultimate
-options:
-  events:
-    - code_suggestions_authenticate
-events:
-  - name: code_suggestions_authenticate

config/metrics/counts_7d/count_total_code_suggestions_authenticate_7d.yml

----
-key_path: count_total_code_suggestions_authenticate_7d
-description: Tracks the number of authentication requests to the `POST /code_suggestions/tokens` API endpoint in the last 7 days.
-product_section: dev
-product_stage: create
-product_group: code_creation
-performance_indicator_type: []
-value_type: number
-status: active
-milestone: "16.8"
-introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/138848
-time_frame: 7d
-data_source: internal_events
-data_category: optional
-distribution:
-- ce
-- ee
-tier:
-- free
-- premium
-- ultimate
-options:
-  events:
-    - code_suggestions_authenticate
-events:
-  - name: code_suggestions_authenticate

doc/api/code_suggestions.md

@@ -8,32 +8,6 @@ info: To determine the technical writer assigned to the Stage/Group associated w
 
 Use the Code Suggestions API to access the Code Suggestions feature.
 
-## Create an access token
-
-> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/404427) in GitLab 16.1.
-
-Creates an access token to access Code Suggestions.
-
-```plaintext
-POST /code_suggestions/tokens
-```
-
-```shell
-curl --request POST \
-  --header "PRIVATE-TOKEN: <your_access_token>" \
-  --url "https://gitlab.example.com/api/v4/code_suggestions/tokens"
-```
-
-Example response:
-
-```json
-{
-    "access_token": "secret-access-token",
-    "expires_in": 3600,
-    "created_at": 1687865199
-}
-```
-
 ## Generate code completions
 
 DETAILS:

ee/lib/api/code_suggestions.rb

@@ -50,36 +50,6 @@ def saas_headers
     end
 
     namespace 'code_suggestions' do
-      resources :tokens do
-        desc 'Create an access token' do
-          detail 'Creates an access token to access Code Suggestions.'
-          success Entities::CodeSuggestionsAccessToken
-          failure [
-            { code: 401, message: 'Unauthorized' },
-            { code: 404, message: 'Not found' }
-          ]
-        end
-        post do
-          not_found! unless Gitlab.org_or_com?
-
-          Gitlab::Tracking.event(
-            'API::CodeSuggestions',
-            :authenticate,
-            user: current_user,
-            label: 'code_suggestions'
-          )
-
-          Gitlab::InternalEvents.track_event(
-            'code_suggestions_authenticate',
-            user: current_user
-          )
-
-          token = Gitlab::CloudConnector::SelfIssuedToken.new(
-            current_user, scopes: [:code_suggestions], gitlab_realm: gitlab_realm)
-          present token, with: Entities::CodeSuggestionsAccessToken
-        end
-      end
-
       resources :completions do
         params do
           requires :current_file, type: Hash do

ee/spec/requests/api/code_suggestions_spec.rb

@@ -133,61 +133,6 @@
     end
   end
 
-  describe 'POST /code_suggestions/tokens' do
-    subject(:post_api) { post api('/code_suggestions/tokens', current_user), headers: headers }
-
-    context 'when user is not logged in' do
-      let(:current_user) { nil }
-
-      include_examples 'an unauthorized response'
-
-      context 'and access token is provided' do
-        it_behaves_like 'an endpoint authenticated with token'
-      end
-    end
-
-    context 'when user is logged in' do
-      let(:current_user) { authorized_user }
-
-      context 'when API feature flag is disabled' do
-        before do
-          stub_feature_flags(code_suggestions_tokens_api: false)
-        end
-
-        include_examples 'a not found response'
-      end
-
-      context 'with no access to code suggestions' do
-        let(:access_code_suggestions) { false }
-
-        include_examples 'an unauthorized response'
-      end
-
-      context 'with access to code suggestions' do
-        context 'when on .org or .com' do
-          include_examples 'a successful response'
-          it_behaves_like 'an endpoint authenticated with token'
-
-          it 'sets the access token realm to SaaS' do
-            expect(Gitlab::CloudConnector::SelfIssuedToken).to receive(:new).with(
-              current_user,
-              scopes: [:code_suggestions],
-              gitlab_realm: Gitlab::CloudConnector::SelfIssuedToken::GITLAB_REALM_SAAS
-            )
-
-            post_api
-          end
-        end
-
-        context 'when not on .org and .com' do
-          let(:is_saas) { false }
-
-          include_examples 'a not found response'
-        end
-      end
-    end
-  end
-
   describe 'POST /code_suggestions/completions' do
     let(:access_code_suggestions) { true }
     let(:global_instance_id) { 'instance-ABC' }