feature: custom alerting

Created by: junotx

What would you like to be added:

Support to custom alerting with prometheus style

Why is this needed:

friendly and user-customized alerting for monitoring metrics easier integration into downstream alert manager and notification manager

Reference

• Design https://github.com/kubesphere/community/blob/master/sig-observability/concepts-and-designs/custom%20alerting.md
• APIs

{
	"swagger": "2.0",
	"info": {
		"description": "KubeSphere OpenAPI",
		"title": "KubeSphere",
		"contact": {
			"name": "KubeSphere",
			"url": "https://kubesphere.io/",
			"email": "kubesphere@yunify.com"
		},
		"license": {
			"name": "Apache 2.0",
			"url": "https://www.apache.org/licenses/LICENSE-2.0.html"
		},
		"version": "0.1.0",
		"x-taggroups": [
			{
				"name": "Custom Alerting",
				"tags": [
					"Alerting"
				]
			}
		]
	},
	"paths": {
		"/kapis/alerting.kubesphere.io/v2alpha1/alerts": {
			"get": {
				"produces": [
					"application/json"
				],
				"tags": [
					"Alerting"
				],
				"summary": "list the alerts of the cluster-level custom alerting rules",
				"operationId": "handleListCustomRulesAlerts",
				"parameters": [
					{
						"type": "string",
						"description": "state, one of `firing`, `pending`, `inactive`",
						"name": "state",
						"in": "query"
					},
					{
						"type": "string",
						"format": "key=%s,key~%s",
						"description": "label filters, concatenating multiple filters with commas, equal symbol for exact query, wave symbol for fuzzy query e.g. name~a",
						"name": "label_filters",
						"in": "query"
					},
					{
						"type": "integer",
						"default": 0,
						"description": "offset of the result set",
						"name": "offset",
						"in": "query"
					},
					{
						"type": "integer",
						"default": 10,
						"description": "limit size of the result set",
						"name": "limit",
						"in": "query"
					}
				],
				"responses": {
					"200": {
						"description": "ok",
						"schema": {
							"$ref": "#/definitions/v2alpha1.AlertList"
						}
					}
				}
			}
		},
		"/kapis/alerting.kubesphere.io/v2alpha1/builtin/alerts": {
			"get": {
				"produces": [
					"application/json"
				],
				"tags": [
					"Alerting"
				],
				"summary": "list the alerts of the builtin(non-custom) rules",
				"operationId": "handleListBuiltinRulesAlerts",
				"parameters": [
					{
						"type": "string",
						"description": "state, one of `firing`, `pending`, `inactive`",
						"name": "state",
						"in": "query"
					},
					{
						"type": "string",
						"format": "key=%s,key~%s",
						"description": "label filters, concatenating multiple filters with commas, equal symbol for exact query, wave symbol for fuzzy query e.g. name~a",
						"name": "label_filters",
						"in": "query"
					},
					{
						"type": "integer",
						"default": 0,
						"description": "offset of the result set",
						"name": "offset",
						"in": "query"
					},
					{
						"type": "integer",
						"default": 10,
						"description": "limit size of the result set",
						"name": "limit",
						"in": "query"
					}
				],
				"responses": {
					"200": {
						"description": "ok",
						"schema": {
							"$ref": "#/definitions/v2alpha1.AlertList"
						}
					}
				}
			}
		},
		"/kapis/alerting.kubesphere.io/v2alpha1/builtin/rules": {
			"get": {
				"produces": [
					"application/json"
				],
				"tags": [
					"Alerting"
				],
				"summary": "list the builtin(non-custom) alerting rules",
				"operationId": "handleListBuiltinAlertingRules",
				"parameters": [
					{
						"type": "string",
						"description": "rule name",
						"name": "name",
						"in": "query"
					},
					{
						"type": "string",
						"description": "state of a rule based on its alerts, one of `firing`, `pending`, `inactive`",
						"name": "state",
						"in": "query"
					},
					{
						"type": "string",
						"description": "health state of a rule based on the last execution, one of `ok`, `err`, `unknown`",
						"name": "health",
						"in": "query"
					},
					{
						"type": "string",
						"format": "key=%s,key~%s",
						"description": "label filters, concatenating multiple filters with commas, equal symbol for exact query, wave symbol for fuzzy query e.g. name~a",
						"name": "label_filters",
						"in": "query"
					},
					{
						"type": "string",
						"description": "sort field, one of `name`, `lastEvaluation`, `evaluationTime`",
						"name": "sort_field",
						"in": "query"
					},
					{
						"type": "string",
						"description": "sort type, one of `asc`, `desc`",
						"name": "sort_type",
						"in": "query"
					},
					{
						"type": "integer",
						"default": 0,
						"description": "offset of the result set",
						"name": "offset",
						"in": "query"
					},
					{
						"type": "integer",
						"default": 10,
						"description": "limit size of the result set",
						"name": "limit",
						"in": "query"
					}
				],
				"responses": {
					"200": {
						"description": "ok",
						"schema": {
							"$ref": "#/definitions/v2alpha1.GettableAlertingRuleList"
						}
					}
				}
			}
		},
		"/kapis/alerting.kubesphere.io/v2alpha1/builtin/rules/{rule_id}": {
			"get": {
				"produces": [
					"application/json"
				],
				"tags": [
					"Alerting"
				],
				"summary": "get the builtin(non-custom) alerting rule with specified id",
				"operationId": "handleGetBuiltinAlertingRule",
				"responses": {
					"200": {
						"description": "ok",
						"schema": {
							"$ref": "#/definitions/v2alpha1.GettableAlertingRule"
						}
					}
				}
			}
		},
		"/kapis/alerting.kubesphere.io/v2alpha1/builtin/rules/{rule_id}/alerts": {
			"get": {
				"produces": [
					"application/json"
				],
				"tags": [
					"Alerting"
				],
				"summary": "list the alerts of the builtin(non-custom) alerting rule with the specified id",
				"operationId": "handleListBuiltinRuleAlerts",
				"responses": {
					"200": {
						"description": "ok",
						"schema": {
							"type": "array",
							"items": {
								"$ref": "#/definitions/v2alpha1.Alert"
							}
						}
					}
				}
			}
		},
		"/kapis/alerting.kubesphere.io/v2alpha1/namespaces/{namespace}/alerts": {
			"get": {
				"produces": [
					"application/json"
				],
				"tags": [
					"Alerting"
				],
				"summary": "list the alerts of the custom alerting rules in the specified namespace.",
				"operationId": "handleListCustomRulesAlerts",
				"parameters": [
					{
						"type": "string",
						"description": "state, one of `firing`, `pending`, `inactive`",
						"name": "state",
						"in": "query"
					},
					{
						"type": "string",
						"format": "key=%s,key~%s",
						"description": "label filters, concatenating multiple filters with commas, equal symbol for exact query, wave symbol for fuzzy query e.g. name~a",
						"name": "label_filters",
						"in": "query"
					},
					{
						"type": "integer",
						"default": 0,
						"description": "offset of the result set",
						"name": "offset",
						"in": "query"
					},
					{
						"type": "integer",
						"default": 10,
						"description": "limit size of the result set",
						"name": "limit",
						"in": "query"
					}
				],
				"responses": {
					"200": {
						"description": "ok",
						"schema": {
							"$ref": "#/definitions/v2alpha1.AlertList"
						}
					}
				}
			}
		},
		"/kapis/alerting.kubesphere.io/v2alpha1/namespaces/{namespace}/rules": {
			"get": {
				"produces": [
					"application/json"
				],
				"tags": [
					"Alerting"
				],
				"summary": "list the custom alerting rules in the specified namespace",
				"operationId": "handleListCustomAlertingRules",
				"parameters": [
					{
						"type": "string",
						"description": "rule name",
						"name": "name",
						"in": "query"
					},
					{
						"type": "string",
						"description": "state of a rule based on its alerts, one of `firing`, `pending`, `inactive`",
						"name": "state",
						"in": "query"
					},
					{
						"type": "string",
						"description": "health state of a rule based on the last execution, one of `ok`, `err`, `unknown`",
						"name": "health",
						"in": "query"
					},
					{
						"type": "string",
						"format": "key=%s,key~%s",
						"description": "label filters, concatenating multiple filters with commas, equal symbol for exact query, wave symbol for fuzzy query e.g. name~a",
						"name": "label_filters",
						"in": "query"
					},
					{
						"type": "string",
						"description": "sort field, one of `name`, `lastEvaluation`, `evaluationTime`",
						"name": "sort_field",
						"in": "query"
					},
					{
						"type": "string",
						"description": "sort type, one of `asc`, `desc`",
						"name": "sort_type",
						"in": "query"
					},
					{
						"type": "integer",
						"default": 0,
						"description": "offset of the result set",
						"name": "offset",
						"in": "query"
					},
					{
						"type": "integer",
						"default": 10,
						"description": "limit size of the result set",
						"name": "limit",
						"in": "query"
					}
				],
				"responses": {
					"200": {
						"description": "ok",
						"schema": {
							"$ref": "#/definitions/v2alpha1.GettableAlertingRuleList"
						}
					}
				}
			},
			"post": {
				"produces": [
					"application/json"
				],
				"tags": [
					"Alerting"
				],
				"summary": "create a custom alerting rule in the specified namespace",
				"operationId": "handleCreateCustomAlertingRule",
				"parameters": [
					{
						"name": "body",
						"in": "body",
						"required": true,
						"schema": {
							"$ref": "#/definitions/v2alpha1.PostableAlertingRule"
						}
					}
				],
				"responses": {
					"200": {
						"description": "ok",
						"schema": {
							"type": "string"
						}
					}
				}
			}
		},
		"/kapis/alerting.kubesphere.io/v2alpha1/namespaces/{namespace}/rules/{rule_name}": {
			"get": {
				"produces": [
					"application/json"
				],
				"tags": [
					"Alerting"
				],
				"summary": "get the custom alerting rule with the specified name in the specified namespace",
				"operationId": "handleGetCustomAlertingRule",
				"responses": {
					"200": {
						"description": "ok",
						"schema": {
							"$ref": "#/definitions/v2alpha1.GettableAlertingRule"
						}
					}
				}
			},
			"put": {
				"produces": [
					"application/json"
				],
				"tags": [
					"Alerting"
				],
				"summary": "update the custom alerting rule with the specified name in the specified namespace",
				"operationId": "handleUpdateCustomAlertingRule",
				"parameters": [
					{
						"name": "body",
						"in": "body",
						"required": true,
						"schema": {
							"$ref": "#/definitions/v2alpha1.PostableAlertingRule"
						}
					}
				],
				"responses": {
					"200": {
						"description": "ok",
						"schema": {
							"type": "string"
						}
					}
				}
			},
			"delete": {
				"produces": [
					"application/json"
				],
				"tags": [
					"Alerting"
				],
				"summary": "delete the custom alerting rule with the specified rule name in the specified namespace",
				"operationId": "handleDeleteCustomAlertingRule",
				"responses": {
					"200": {
						"description": "ok"
					}
				}
			}
		},
		"/kapis/alerting.kubesphere.io/v2alpha1/namespaces/{namespace}/rules/{rule_name}/alerts": {
			"get": {
				"produces": [
					"application/json"
				],
				"tags": [
					"Alerting"
				],
				"summary": "get the alerts of the custom alerting rule with the specified name in the specified namespace",
				"operationId": "handleListCustomRuleAlerts",
				"responses": {
					"200": {
						"description": "ok",
						"schema": {
							"type": "array",
							"items": {
								"$ref": "#/definitions/v2alpha1.Alert"
							}
						}
					}
				}
			}
		},
		"/kapis/alerting.kubesphere.io/v2alpha1/rules": {
			"get": {
				"produces": [
					"application/json"
				],
				"tags": [
					"Alerting"
				],
				"summary": "list the cluster-level custom alerting rules",
				"operationId": "handleListCustomAlertingRules",
				"parameters": [
					{
						"type": "string",
						"description": "rule name",
						"name": "name",
						"in": "query"
					},
					{
						"type": "string",
						"description": "state of a rule based on its alerts, one of `firing`, `pending`, `inactive`",
						"name": "state",
						"in": "query"
					},
					{
						"type": "string",
						"description": "health state of a rule based on the last execution, one of `ok`, `err`, `unknown`",
						"name": "health",
						"in": "query"
					},
					{
						"type": "string",
						"format": "key=%s,key~%s",
						"description": "label filters, concatenating multiple filters with commas, equal symbol for exact query, wave symbol for fuzzy query e.g. name~a",
						"name": "label_filters",
						"in": "query"
					},
					{
						"type": "string",
						"description": "sort field, one of `name`, `lastEvaluation`, `evaluationTime`",
						"name": "sort_field",
						"in": "query"
					},
					{
						"type": "string",
						"description": "sort type, one of `asc`, `desc`",
						"name": "sort_type",
						"in": "query"
					},
					{
						"type": "integer",
						"default": 0,
						"description": "offset of the result set",
						"name": "offset",
						"in": "query"
					},
					{
						"type": "integer",
						"default": 10,
						"description": "limit size of the result set",
						"name": "limit",
						"in": "query"
					}
				],
				"responses": {
					"200": {
						"description": "ok",
						"schema": {
							"$ref": "#/definitions/v2alpha1.GettableAlertingRuleList"
						}
					}
				}
			},
			"post": {
				"produces": [
					"application/json"
				],
				"tags": [
					"Alerting"
				],
				"summary": "create a cluster-level custom alerting rule",
				"operationId": "handleCreateCustomAlertingRule",
				"parameters": [
					{
						"name": "body",
						"in": "body",
						"required": true,
						"schema": {
							"$ref": "#/definitions/v2alpha1.PostableAlertingRule"
						}
					}
				],
				"responses": {
					"200": {
						"description": "ok"
					}
				}
			}
		},
		"/kapis/alerting.kubesphere.io/v2alpha1/rules/{rule_name}": {
			"get": {
				"produces": [
					"application/json"
				],
				"tags": [
					"Alerting"
				],
				"summary": "get the cluster-level custom alerting rule with the specified name",
				"operationId": "handleGetCustomAlertingRule",
				"responses": {
					"200": {
						"description": "ok",
						"schema": {
							"$ref": "#/definitions/v2alpha1.GettableAlertingRule"
						}
					}
				}
			},
			"put": {
				"produces": [
					"application/json"
				],
				"tags": [
					"Alerting"
				],
				"summary": "update the cluster-level custom alerting rule with the specified name",
				"operationId": "handleUpdateCustomAlertingRule",
				"parameters": [
					{
						"name": "body",
						"in": "body",
						"required": true,
						"schema": {
							"$ref": "#/definitions/v2alpha1.PostableAlertingRule"
						}
					}
				],
				"responses": {
					"200": {
						"description": "ok"
					}
				}
			},
			"delete": {
				"produces": [
					"application/json"
				],
				"tags": [
					"Alerting"
				],
				"summary": "delete the cluster-level custom alerting rule with the specified name",
				"operationId": "handleDeleteCustomAlertingRule",
				"responses": {
					"200": {
						"description": "ok"
					}
				}
			}
		},
		"/kapis/alerting.kubesphere.io/v2alpha1/rules/{rule_name}/alerts": {
			"get": {
				"produces": [
					"application/json"
				],
				"tags": [
					"Alerting"
				],
				"summary": "list the alerts of the cluster-level custom alerting rule with the specified name",
				"operationId": "handleListCustomRuleAlerts",
				"responses": {
					"200": {
						"description": "ok",
						"schema": {
							"type": "array",
							"items": {
								"$ref": "#/definitions/v2alpha1.Alert"
							}
						}
					}
				}
			}
		}
	},
	"definitions": {
		"v2alpha1.Alert": {
			"properties": {
				"activeAt": {
					"description": "time when alert is active",
					"type": "string",
					"format": "date-time"
				},
				"annotations": {
					"description": "annotations",
					"type": "object",
					"additionalProperties": {
						"type": "string"
					}
				},
				"labels": {
					"description": "labels",
					"type": "object",
					"additionalProperties": {
						"type": "string"
					}
				},
				"ruleId": {
					"description": "rule id triggering the alert",
					"type": "string"
				},
				"ruleName": {
					"description": "rule name triggering the alert",
					"type": "string"
				},
				"state": {
					"description": "state",
					"type": "string"
				},
				"value": {
					"description": "the value at the last evaluation of the query expression",
					"type": "string"
				}
			}
		},
		"v2alpha1.AlertList": {
			"required": [
				"items",
				"total"
			],
			"properties": {
				"items": {
					"type": "array",
					"items": {
						"$ref": "#/definitions/v2alpha1.Alert"
					}
				},
				"total": {
					"type": "integer",
					"format": "int32"
				}
			}
		},
		"v2alpha1.GettableAlertingRule": {
			"properties": {
				"alerts": {
					"description": "alerts",
					"type": "array",
					"items": {
						"$ref": "#/definitions/v2alpha1.Alert"
					}
				},
				"annotations": {
					"description": "non-identifying key/value pairs. summary, message, description are the commonly used annotation names",
					"type": "object",
					"additionalProperties": {
						"type": "string"
					}
				},
				"duration": {
					"description": "duration an alert transitions from Pending to Firing state, which must match ^([0-9]+)(y|w|d|h|m|s|ms)$",
					"type": "string"
				},
				"evaluationTime": {
					"description": "taken seconds for evaluation of query expression",
					"type": "number",
					"format": "double"
				},
				"health": {
					"description": "health state of a rule based on the last execution, one of ok, err, unknown",
					"type": "string"
				},
				"id": {
					"description": "rule id is only used by built-in alerting rules",
					"type": "string"
				},
				"labels": {
					"description": "extra labels to attach to the resulting alert sample vectors (the key string has to match [a-zA-Z_][a-zA-Z0-9_]*). eg: a typical label called severity, whose value may be info, warning, error, critical, is usually used to indicate the severity of an alert",
					"type": "object",
					"additionalProperties": {
						"type": "string"
					}
				},
				"lastError": {
					"description": "error for the last execution",
					"type": "string"
				},
				"lastEvaluation": {
					"description": "time for last evaluation of query expression",
					"type": "string",
					"format": "date-time"
				},
				"name": {
					"description": "rule name should be unique in one namespace for custom alerting rules",
					"type": "string"
				},
				"query": {
					"description": "prometheus query expression, grammars of which may be referred to https://prometheus.io/docs/prometheus/latest/querying/basics/",
					"type": "string"
				},
				"state": {
					"description": "state of a rule based on its alerts, one of firing, pending, inactive",
					"type": "string"
				}
			}
		},
		"v2alpha1.GettableAlertingRuleList": {
			"required": [
				"items",
				"total"
			],
			"properties": {
				"items": {
					"type": "array",
					"items": {
						"$ref": "#/definitions/v2alpha1.GettableAlertingRule"
					}
				},
				"total": {
					"type": "integer",
					"format": "int32"
				}
			}
		},
		"v2alpha1.PostableAlertingRule": {
			"properties": {
				"annotations": {
					"description": "non-identifying key/value pairs. summary, message, description are the commonly used annotation names",
					"type": "object",
					"additionalProperties": {
						"type": "string"
					}
				},
				"duration": {
					"description": "duration an alert transitions from Pending to Firing state, which must match ^([0-9]+)(y|w|d|h|m|s|ms)$",
					"type": "string"
				},
				"id": {
					"description": "rule id is only used by built-in alerting rules",
					"type": "string"
				},
				"labels": {
					"description": "extra labels to attach to the resulting alert sample vectors (the key string has to match [a-zA-Z_][a-zA-Z0-9_]*). eg: a typical label called severity, whose value may be info, warning, error, critical, is usually used to indicate the severity of an alert",
					"type": "object",
					"additionalProperties": {
						"type": "string"
					}
				},
				"name": {
					"description": "rule name should be unique in one namespace for custom alerting rules",
					"type": "string"
				},
				"query": {
					"description": "prometheus query expression, grammars of which may be referred to https://prometheus.io/docs/prometheus/latest/querying/basics/",
					"type": "string"
				}
			}
		}
	},
	"securityDefinitions": {
		"jwt": {
			"type": "apiKey",
			"name": "Authorization",
			"in": "header"
		}
	},
	"security": [
		{
			"jwt": []
		}
	]
}

/area alerting /kind feature /assign @leoendless