ci-tasks/deployment-project/runway.yml

@@ -63,7 +63,12 @@ variables:
     # TODO: REPLACE WITH NEEDS PIPELINE JOB ONCE https://gitlab.com/gitlab-org/gitlab/-/issues/331596 is fixed
     - curl -f -s -S --location --output artifacts.zip --header "JOB-TOKEN:$CI_JOB_TOKEN" "${CI_API_V4_URL}/projects/${SOURCE_PROJECT_ID}/jobs/${PARENT_ARTIFACTS_JOB_ID}/artifacts"
     - unzip artifacts.zip
-    - export TF_VAR_stable_revision=$(runwayctl reconciler output -- $ENVIRONMENT -json | jq -r .stable_revision.value)
+    - >
+      if [[ -n "$FORCE_REVISION" ]]; then
+          echo "Forcing new stable revision"
+        else
+          export TF_VAR_stable_revisions=$(runwayctl reconciler output -- $ENVIRONMENT -json | jq -r .stable_revisions.value)
+        fi
     - runwayctl validate
     # Apply terraform
     - runwayctl reconciler apply ${ENVIRONMENT}
@@ -155,15 +160,18 @@ variables:
   script:
     - export VAULT_TOKEN=$(vault write -field=token "auth/${VAULT_AUTH_PATH}/login" role="${VAULT_AUTH_ROLE}" jwt="${VAULT_ID_TOKEN}")
     - export VAULT_IMPERSONATED_TOKEN=$(vault read -field=token "gcp/impersonated-account/${PROMETHEUS_IAP_GOOGLE_PROJECT}--${PROMETHEUS_IAP_SERVICE_ACCOUNT}/token")
-    - export STABLE_REVISION=$(runwayctl reconciler output -- $ENVIRONMENT -json | jq -r .stable_revision.value)
+    - export STABLE_REVISIONS=$(runwayctl reconciler output -- $ENVIRONMENT -json | jq -r .stable_revisions.value)
     - |
-      if runwayctl monitor "${ENVIRONMENT}" "${STABLE_REVISION}"; then
-        echo "Success: monitor passed."
-        exit 0
-      elif [[ -n "$PROMOTE_OVERRIDE_REASON" ]]; then
-        echo "Warning: monitor detected 5xx but overridden due to: $PROMOTE_OVERRIDE_REASON"
-        exit 0
-      else
-        echo "Failure: elevated error rate for revision $STABLE_REVISION. Set $PROMOTE_OVERRIDE_REASON as CI variable to override."
-        exit 1
-      fi
+      echo $STABLE_REVISIONS | jq -r 'to_entries[] | "\(.key) \(.value)"' | while read -r region revision; do
+        if runwayctl monitor "$ENVIRONMENT" "${region}" "${revision}"; then
+          echo "Success: monitor passed."
+          continue
+        elif [[ -n "$PROMOTE_OVERRIDE_REASON" ]]; then
+          echo "Warning: monitor detected 5xx but overridden due to: $PROMOTE_OVERRIDE_REASON"
+          continue
+        else
+          echo "Failure: elevated error rate for region="${region}" revision="${revision}". Set `PROMOTE_OVERRIDE_REASON` as CI variable to override."
+          break
+        fi
+      done
+      exit 0

cmd/monitor/monitor.go

@@ -45,16 +45,16 @@ var (
 // NewMonitorCmd returns new cobra command.
 func NewMonitorCmd() *cobra.Command {
 	cmd := &cobra.Command{
-		Use:   "monitor <environment> <revision_name>",
+		Use:   "monitor <environment> <region> <revision_name>",
 		Short: "Monitor service",
-		Args:  checkargs.WithUsage(cobra.ExactArgs(2)),
+		Args:  checkargs.WithUsage(cobra.ExactArgs(3)),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			env, ok := Environment(args[0])
 			if !ok {
 				return ErrInvalidEnv
 			}
 
-			return monitor(cmd.Context(), env, args[1])
+			return monitor(cmd.Context(), env, args[1], args[2])
 		},
 	}
 
@@ -98,7 +98,7 @@ func (rv *DataPoint) UnmarshalJSON(b []byte) error {
 	return nil
 }
 
-func monitor(ctx context.Context, env, revisionName string) error {
+func monitor(ctx context.Context, env, region, revisionName string) error {
 	prometheusClient, err := newPrometheusClient(ctx)
 	if err != nil {
 		return err
@@ -117,7 +117,7 @@ func monitor(ctx context.Context, env, revisionName string) error {
 		timeoutCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
 		defer cancel()
 
-		result, err := queryThanos(timeoutCtx, prometheusClient, env, revisionName)
+		result, err := queryThanos(timeoutCtx, prometheusClient, env, region, revisionName)
 		if err != nil {
 			return err
 		}
@@ -182,8 +182,8 @@ func validateResults(results []Result) bool {
 	return true
 }
 
-func queryThanos(ctx context.Context, client *http.Client, env, revisionName string) (*QueryResult, error) {
-	query := buildPrometheusQuery(env, revisionName)
+func queryThanos(ctx context.Context, client *http.Client, env, region, revisionName string) (*QueryResult, error) {
+	query := buildPrometheusQuery(env, region, revisionName)
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, query, nil)
 	if err != nil {
@@ -221,12 +221,14 @@ func sendRequest(client *http.Client, req *http.Request) ([]byte, error) {
 	return b, nil
 }
 
-func buildPrometheusQuery(env, revisionName string) string {
+func buildPrometheusQuery(env, region, revisionName string) string {
 	query := fmt.Sprintf(
 		`(
 			sum(
 				avg_over_time(
-					stackdriver_cloud_run_revision_run_googleapis_com_request_count{response_code_class="5xx",env=%q,revision_name=%q}[1m] offset 30s
+					stackdriver_cloud_run_revision_run_googleapis_com_request_count{
+						response_code_class="5xx",env=%q,location=%q,revision_name=%q
+					}[1m] offset 30s
 				) / 60
 			)
 		)
@@ -234,11 +236,11 @@ func buildPrometheusQuery(env, revisionName string) string {
 		(
 			sum(
 				avg_over_time(
-					stackdriver_cloud_run_revision_run_googleapis_com_request_count{env=%q,revision_name=%q}[1m] offset 30s
+					stackdriver_cloud_run_revision_run_googleapis_com_request_count{env=%q,location=%q,revision_name=%q}[1m] offset 30s
 				) / 60
 			)
 		)
-		`, env, revisionName, env, revisionName,
+		`, env, region, revisionName, env, region, revisionName,
 	)
 
 	fmt.Println("Running query: " + query)

reconciler/loadbalancers.tf

@@ -12,13 +12,14 @@ data "google_dns_managed_zone" "runway" {
 }
 
 resource "google_compute_region_network_endpoint_group" "serverless_neg" {
+  for_each              = local.regions
   provider              = google-beta
   name                  = "${var.runway_service_id}-neg"
   project               = var.project
   network_endpoint_type = "SERVERLESS"
-  region                = var.region
+  region                = each.key
   cloud_run {
-    service = google_cloud_run_v2_service.runway_service.name
+    service = google_cloud_run_v2_service.runway_service[each.key].name
   }
 }
 
@@ -27,14 +28,13 @@ module "external_lb" {
 
   source = "./modules/external-loadbalancer"
 
-  project = var.project
-
+  project                      = var.project
   runway_ssl_policy            = google_compute_ssl_policy.runway_tls_1_2.self_link
   runway_service_id            = var.runway_service_id
   runway_service_dns_name      = local.service_dns_name
   runway_lb_security_policy_id = google_compute_security_policy.runway_lb.id
   runway_dns_managed_zone_name = data.google_dns_managed_zone.runway.name
-  runway_serverless_neg_id     = google_compute_region_network_endpoint_group.serverless_neg.id
+  runway_serverless_neg_ids    = [for index, _region in local.regions : google_compute_region_network_endpoint_group.serverless_neg[index].id]
 
   resource_labels = local.resource_labels
 }
@@ -45,15 +45,14 @@ module "internal_lb" {
   source = "./modules/internal-loadbalancer"
 
   project     = var.project
-  region      = var.region
+  regions     = local.regions
   environment = var.environment
 
   runway_ssl_policy            = google_compute_ssl_policy.runway_tls_1_2.self_link
   runway_service_id            = var.runway_service_id
   runway_service_dns_name      = local.service_dns_name_internal
   runway_dns_managed_zone_name = data.google_dns_managed_zone.runway.name
-  runway_serverless_neg_id     = google_compute_region_network_endpoint_group.serverless_neg.id
+  runway_serverless_neg_ids    = [for index, _region in local.regions : google_compute_region_network_endpoint_group.serverless_neg[index].id]
 
   resource_labels = local.resource_labels
 }
-

reconciler/main.tf

@@ -21,11 +21,12 @@ locals {
   )
   runway_yml                = try(yamldecode(data.local_file.runway_yml.content), {})
   gcr_image                 = "${var.region}-docker.pkg.dev/${var.project}/${var.runway_service_id}/${element(split("/", var.image), length(split("/", var.image)) - 1)}"
-  stable_revision_traffic   = (var.stable_revision == "null" || var.canary_percent == 100) ? { revision = "${var.runway_service_id}-${random_string.revision.result}" } : { revision = var.stable_revision }
+  create_stable_revision    = (var.stable_revisions == {} || var.canary_percent == 100)
   service_dns_name          = "${var.runway_service_id}.${data.google_dns_managed_zone.runway.dns_name}"
   service_dns_name_internal = "${var.runway_service_id}.internal.${data.google_dns_managed_zone.runway.dns_name}"
   deploy_external_lb        = tobool(try(local.runway_yml.spec.load_balancing.external_load_balancer.enabled, true))
   deploy_internal_lb        = tobool(try(local.runway_yml.spec.load_balancing.internal_load_balancer.enabled, false))
+  regions                   = toset(sort(try(local.runway_yml.spec.regions, ["us-east1"])))
   scrape_targets            = try(local.runway_yml.spec.observability.scrape_targets, [])
   enable_otel_collector     = length(local.scrape_targets) > 0
   apply_cloudflare_policy   = try(local.runway_yml.spec.network_policies.cloudflare, false)
@@ -74,6 +75,7 @@ resource "random_string" "revision" {
     env_vars_file   = data.local_file.env_vars.content
     cpu             = try(local.runway_yml.spec.resources.limits.cpu, "")
     memory          = try(local.runway_yml.spec.resources.limits.memory, "")
+    regions         = join(",", local.regions)
     scrape_targets  = join(",", local.scrape_targets)
     secrets_sha     = sha512(join("", [for secret in local.filtered_secrets : "${secret}.${google_secret_manager_secret_version.vault_secret_version_data[secret].secret_data}"]))
   }
@@ -105,9 +107,10 @@ resource "google_secret_manager_secret_version" "secret_version_data" {
 }
 
 resource "google_cloud_run_v2_service" "runway_service" {
+  for_each     = local.regions
   provider     = google-beta
   name         = var.runway_service_id
-  location     = var.region
+  location     = each.key
   ingress      = local.deploy_external_lb ? "INGRESS_TRAFFIC_INTERNAL_LOAD_BALANCER" : "INGRESS_TRAFFIC_INTERNAL_ONLY"
   launch_stage = "BETA"
   labels       = local.resource_labels
@@ -250,7 +253,7 @@ resource "google_cloud_run_v2_service" "runway_service" {
   }
 
   dynamic "traffic" {
-    for_each = local.stable_revision_traffic
+    for_each = local.create_stable_revision || (try(length(var.stable_revisions), 0) != length(local.regions)) ? { revision = "${var.runway_service_id}-${random_string.revision.result}" } : { revision = var.stable_revisions[each.key] }
     content {
       type     = "TRAFFIC_TARGET_ALLOCATION_TYPE_REVISION"
       revision = traffic.value
@@ -275,9 +278,10 @@ data "google_iam_policy" "noauth" {
 }
 
 resource "google_cloud_run_service_iam_policy" "noauth" {
-  location = var.region
+  for_each = local.regions
+  location = each.key
   project  = var.project
-  service  = google_cloud_run_v2_service.runway_service.name
+  service  = google_cloud_run_v2_service.runway_service[each.key].name
 
   policy_data = data.google_iam_policy.noauth.policy_data
 }

reconciler/modules/external-loadbalancer/main.tf

@@ -16,12 +16,8 @@ module "lb_http" {
   backends = {
     default = {
       description = null
-      groups = [
-        {
-          group = var.runway_serverless_neg_id
-        }
-      ]
-      enable_cdn = false
+      groups      = [for _index, id in var.runway_serverless_neg_ids : { group = id }]
+      enable_cdn  = false
 
       iap_config = {
         enable               = false

reconciler/modules/external-loadbalancer/variables.tf

@@ -35,7 +35,7 @@ variable "runway_dns_managed_zone_name" {
   description = "Runway DNS managed zone name"
 }
 
-variable "runway_serverless_neg_id" {
-  type        = string
-  description = "Runway serverless neg ID"
+variable "runway_serverless_neg_ids" {
+  type        = list(string)
+  description = "Runway serverless neg IDs"
 }

reconciler/modules/internal-loadbalancer/main.tf

@@ -3,8 +3,9 @@
 # - staging: https://ops.gitlab.net/gitlab-com/gl-infra/config-mgmt/-/blob/master/environments/runway-staging/network.tf
 # - production: https://ops.gitlab.net/gitlab-com/gl-infra/config-mgmt/-/blob/master/environments/runway-production/network.tf
 data "google_compute_subnetwork" "runway_subnet" {
-  name   = "runway-${var.region}"
-  region = var.region
+  for_each = var.regions
+  name     = "runway-${each.key}"
+  region   = each.key
 }
 
 data "google_compute_network" "runway" {
@@ -12,6 +13,7 @@ data "google_compute_network" "runway" {
 }
 
 resource "google_compute_global_forwarding_rule" "internal_lb" {
+  for_each              = var.regions
   provider              = google-beta
   project               = var.project
   name                  = "${var.runway_service_id}-internal"
@@ -21,7 +23,7 @@ resource "google_compute_global_forwarding_rule" "internal_lb" {
   labels                = var.resource_labels
   load_balancing_scheme = "INTERNAL_MANAGED"
   network               = data.google_compute_network.runway.id
-  subnetwork            = data.google_compute_subnetwork.runway_subnet.self_link
+  subnetwork            = data.google_compute_subnetwork.runway_subnet[each.key].self_link
 }
 
 resource "google_compute_target_https_proxy" "internal_lb" {
@@ -50,8 +52,11 @@ resource "google_compute_backend_service" "default" {
   port_name             = "http"
   load_balancing_scheme = "INTERNAL_MANAGED"
 
-  backend {
-    group = var.runway_serverless_neg_id
+  dynamic "backend" {
+    for_each = var.runway_serverless_neg_ids
+    content {
+      group = each.key
+    }
   }
 
   log_config {
@@ -94,12 +99,13 @@ resource "google_dns_record_set" "internal_acme_challenge" {
 }
 
 resource "google_dns_record_set" "runway_internal" {
-  project = var.project
-  name    = var.runway_service_dns_name
-  type    = "A"
-  ttl     = 300
+  for_each = var.regions
+  project  = var.project
+  name     = var.runway_service_dns_name
+  type     = "A"
+  ttl      = 300
 
   managed_zone = var.runway_dns_managed_zone_name
 
-  rrdatas = [google_compute_global_forwarding_rule.internal_lb.ip_address]
+  rrdatas = [google_compute_global_forwarding_rule.internal_lb[each.key].ip_address]
 }

reconciler/modules/internal-loadbalancer/variables.tf

@@ -3,9 +3,9 @@ variable "project" {
   description = "Google Project ID"
 }
 
-variable "region" {
-  type        = string
-  description = "GCP Region to use"
+variable "regions" {
+  type        = set(string)
+  description = "GCP Regions to use"
 }
 
 variable "environment" {
@@ -40,7 +40,7 @@ variable "runway_dns_managed_zone_name" {
   description = "Runway DNS managed zone name"
 }
 
-variable "runway_serverless_neg_id" {
-  type        = string
-  description = "Runway serverless neg ID"
+variable "runway_serverless_neg_ids" {
+  type        = list(string)
+  description = "Runway serverless neg IDs"
 }

reconciler/moved.tf

@@ -14,6 +14,21 @@ moved {
   to   = module.external_lb[0].google_dns_record_set.runway
 }
 
+moved {
+  from = google_cloud_run_v2_service.runway_service
+  to   = google_cloud_run_v2_service.runway_service["us-east1"]
+}
+
+moved {
+  from = google_cloud_run_service_iam_policy.noauth
+  to   = google_cloud_run_service_iam_policy.noauth["us-east1"]
+}
+
+moved {
+  from = google_compute_region_network_endpoint_group.serverless_neg
+  to   = google_compute_region_network_endpoint_group.serverless_neg["us-east1"]
+}
+
 #
 # Removed resources
 #

reconciler/outputs.tf

-output "latest_ready_revision" {
-  value = split("revisions/", google_cloud_run_v2_service.runway_service.latest_ready_revision)[1]
-}
-
-output "stable_revision" {
-  value = tolist([for each in google_cloud_run_v2_service.runway_service.traffic : each.revision if each.tag == "stable"])[0]
-}
-
-output "cloud_run_uri" {
-  value = google_cloud_run_v2_service.runway_service.uri
+output "stable_revisions" {
+  value       = { for service in google_cloud_run_v2_service.runway_service : service.location => tolist([for each in service.traffic : each.revision if each.tag == "stable"])[0] }
+  description = "The mapping of Cloud Run region to current stable Cloud Run revision"
 }

reconciler/variables.tf

@@ -3,9 +3,10 @@ variable "project" {
   description = "Google Project ID"
 }
 
+# TODO: https://gitlab.com/gitlab-com/gl-infra/platform/runway/team/-/issues/182
 variable "region" {
   type        = string
-  description = "GCP Region to use"
+  description = "GCP Region to use for Provisioner resources"
 }
 
 variable "runway_service_id" {
@@ -39,10 +40,10 @@ variable "canary_percent" {
   description = "Percentage of traffic to send to canary tag"
 }
 
-variable "stable_revision" {
-  type        = string
-  description = "The cloud run revision that is the current running stable revision"
-  default     = "null"
+variable "stable_revisions" {
+  type        = map(string)
+  description = "The mapping of Cloud Run region to current stable Cloud Run revision"
+  default     = {}
 }
 
 variable "dns_zone" {