Alerting Prototype

docker-compose.yml

@@ -2,7 +2,7 @@
 version: '3.7'
 services:
   pmm-managed-server:
-    image: ${PMM_SERVER_IMAGE:-perconalab/pmm-server:dev-latest}
+    image: ${PMM_SERVER_IMAGE:-perconalab/pmm-server-fb:PR-2512-6a9dc51}
     container_name: pmm-managed-server
     hostname: pmm-managed-server
     environment:

go.mod

@@ -36,7 +36,7 @@ require (
 	github.com/minio/minio-go/v7 v7.0.24
 	github.com/percona-platform/dbaas-api v0.0.0-20220110092915-5aacd784d472
 	github.com/percona-platform/saas v0.0.0-20220427162947-f9d246ad0f16
-	github.com/percona/pmm v0.0.0-20220510110703-cd16c3d93199
+	github.com/percona/pmm v0.0.0-20220511163912-402753c099bd
 	github.com/percona/promconfig v0.2.4-0.20211110115058-98687f586f54
 	github.com/pkg/errors v0.9.1
 	github.com/pmezard/go-difflib v1.0.0

go.sum

@@ -468,6 +468,20 @@ github.com/percona-platform/saas v0.0.0-20220427162947-f9d246ad0f16 h1:0fx16uGtl
 github.com/percona-platform/saas v0.0.0-20220427162947-f9d246ad0f16/go.mod h1:gFUwaFp6Ugu5qsBwiOVJYbDlzgZ77tmXdXGO7tG5xVI=
 github.com/percona/pmm v0.0.0-20220510110703-cd16c3d93199 h1:I4n9DeZypB9AHzejhhsS7koajDYhdkpCN0eNet8mNKw=
 github.com/percona/pmm v0.0.0-20220510110703-cd16c3d93199/go.mod h1:k7HS59HPX33tmrSZGiNzUTYuLr0+a49F3BEZ48MAbuo=
+github.com/percona/pmm v0.0.0-20220510161458-d7fc2d07f4e1 h1:sveoDC12XL1RDBBxjrQaOtXPEFn+EbyRkZRz68//pak=
+github.com/percona/pmm v0.0.0-20220510161458-d7fc2d07f4e1/go.mod h1:k7HS59HPX33tmrSZGiNzUTYuLr0+a49F3BEZ48MAbuo=
+github.com/percona/pmm v0.0.0-20220510175445-628b7c5c9832 h1:VN7prPOjSz0pdUeww32OFQlyXmmnY5YdVTw7yMV6vig=
+github.com/percona/pmm v0.0.0-20220510175445-628b7c5c9832/go.mod h1:k7HS59HPX33tmrSZGiNzUTYuLr0+a49F3BEZ48MAbuo=
+github.com/percona/pmm v0.0.0-20220510183008-f59c8d6e971f h1:++568lw7CJTmxTs0vE74cI1+BldiQ4V5YV7PPcfDc9g=
+github.com/percona/pmm v0.0.0-20220510183008-f59c8d6e971f/go.mod h1:k7HS59HPX33tmrSZGiNzUTYuLr0+a49F3BEZ48MAbuo=
+github.com/percona/pmm v0.0.0-20220510184541-44131a333e76 h1:PoQ7NYmGb/0cXxqo7vneNX+XtKjlr4tYM7oUwf6lErk=
+github.com/percona/pmm v0.0.0-20220510184541-44131a333e76/go.mod h1:k7HS59HPX33tmrSZGiNzUTYuLr0+a49F3BEZ48MAbuo=
+github.com/percona/pmm v0.0.0-20220510185929-24585fa1f30f h1:/i1E4qJLcP0yOoTm2phaulx5acu/yMmeVYRQ/OJZidU=
+github.com/percona/pmm v0.0.0-20220510185929-24585fa1f30f/go.mod h1:k7HS59HPX33tmrSZGiNzUTYuLr0+a49F3BEZ48MAbuo=
+github.com/percona/pmm v0.0.0-20220511114410-81c1e57e2f90 h1:UEZr7Q7o68Uo4HeyXNm8dLVoR9URRxzqTvDNBAloYmM=
+github.com/percona/pmm v0.0.0-20220511114410-81c1e57e2f90/go.mod h1:k7HS59HPX33tmrSZGiNzUTYuLr0+a49F3BEZ48MAbuo=
+github.com/percona/pmm v0.0.0-20220511163912-402753c099bd h1:ClIzyKtNLFnVa/bpTJT2lklSmvHBRZ6sCzVSuZRbPd0=
+github.com/percona/pmm v0.0.0-20220511163912-402753c099bd/go.mod h1:k7HS59HPX33tmrSZGiNzUTYuLr0+a49F3BEZ48MAbuo=
 github.com/percona/promconfig v0.2.4-0.20211110115058-98687f586f54 h1:aI1emmycDTGWKsBdxFPKZqohfBbK4y2ta9G4+RX7gVg=
 github.com/percona/promconfig v0.2.4-0.20211110115058-98687f586f54/go.mod h1:Y2uXi5QNk71+ceJHuI9poank+0S1kjxd3K105fXKVkg=
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=

services/alertmanager/alertmanager.go

@@ -23,6 +23,14 @@ import (
 	_ "embed" // for email templates
 	"encoding/hex"
 	"fmt"
+	"github.com/go-openapi/runtime"
+	httptransport "github.com/go-openapi/runtime/client"
+	"github.com/percona/pmm/api/grafana/gclient"
+	alertmanager2 "github.com/percona/pmm/api/grafana/gclient/alertmanager"
+	"github.com/percona/pmm/api/grafana/gmodels"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/metadata"
+	"google.golang.org/grpc/status"
 	"io/ioutil"
 	"net/http"
 	"os"
@@ -33,7 +41,6 @@ import (
 	"time"
 
 	"github.com/AlekSi/pointer"
-	httptransport "github.com/go-openapi/runtime/client"
 	"github.com/go-openapi/strfmt"
 	"github.com/percona/pmm/api/alertmanager/amclient"
 	"github.com/percona/pmm/api/alertmanager/amclient/alert"
@@ -71,6 +78,8 @@ const (
 	CheckFilter = "stt_check=1"
 	// IAFilter represents AlertManager filter for Integrated Alerts.
 	IAFilter = "ia=1"
+
+	ExcludeChecks = "stt_check!=1"
 )
 
 var notificationLabels = []string{
@@ -737,28 +746,69 @@ func (svc *Service) SendAlerts(ctx context.Context, alerts ammodels.PostableAler
 	}
 }
 
+// TODO it was copy-pasted from services/grafana/client.go
+func authHeadersFromContext(ctx context.Context) (http.Header, error) {
+	headers, ok := metadata.FromIncomingContext(ctx)
+	if !ok {
+		return nil, fmt.Errorf("cannot get headers from metadata")
+	}
+	// get authorization from headers.
+	authorizationHeaders := headers.Get("Authorization")
+	cookieHeaders := headers.Get("grpcgateway-cookie")
+	if len(authorizationHeaders) == 0 && len(cookieHeaders) == 0 {
+		return nil, status.Error(codes.Unauthenticated, "Authorization error.")
+	}
+
+	authHeaders := make(http.Header)
+	if len(authorizationHeaders) != 0 {
+		authHeaders.Add("Authorization", authorizationHeaders[0])
+	}
+	if len(cookieHeaders) != 0 {
+		for _, header := range cookieHeaders {
+			authHeaders.Add("Cookie", header)
+		}
+	}
+	return authHeaders, nil
+}
+
 // GetAlerts returns alerts available in alertmanager.
-func (svc *Service) GetAlerts(ctx context.Context, fp *services.FilterParams) ([]*ammodels.GettableAlert, error) {
-	alertParams := alert.NewGetAlertsParams()
+func (svc *Service) GetAlerts(ctx context.Context, fp *services.FilterParams) ([]*gmodels.GettableAlert, error) {
+	alertParams := alertmanager2.NewRouteGetAMAlertsParams()
 	alertParams.Context = ctx
+	alertParams.Recipient = "grafana"
 
 	if fp != nil {
+		if fp.ExcludeChecks {
+			alertParams.Matchers = append(alertParams.Matchers, ExcludeChecks)
+		}
 		if fp.IsCheck {
-			alertParams.Filter = append(alertParams.Filter, CheckFilter)
+			alertParams.Matchers = append(alertParams.Matchers, CheckFilter)
 		}
 		if fp.IsIA {
-			alertParams.Filter = append(alertParams.Filter, IAFilter)
+			alertParams.Matchers = append(alertParams.Matchers, IAFilter)
 		}
 		if fp.ServiceID != "" {
-			alertParams.Filter = append(alertParams.Filter, fmt.Sprintf("service_id=\"%s\"", fp.ServiceID))
+			alertParams.Matchers = append(alertParams.Matchers, fmt.Sprintf("service_id=\"%s\"", fp.ServiceID))
 		}
 		if fp.AlertID != "" {
-			alertParams.Filter = append(alertParams.Filter, fmt.Sprintf("alert_id=\"%s\"", fp.AlertID))
+			alertParams.Matchers = append(alertParams.Matchers, fmt.Sprintf("alert_id=\"%s\"", fp.AlertID))
 		}
 	}
 
 	svc.l.Debugf("%+v", alertParams)
-	resp, err := amclient.Default.Alert.GetAlerts(alertParams)
+	authHeaders, err := authHeadersFromContext(ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	auth := runtime.ClientAuthInfoWriterFunc(func(r runtime.ClientRequest, reg strfmt.Registry) error {
+		for header, values := range authHeaders {
+			_ = r.SetHeaderParam(header, values...)
+		}
+		return nil
+	})
+
+	resp, err := gclient.Default.Alertmanager.RouteGetAMAlerts(alertParams, auth)
 	if err != nil {
 		return nil, err
 	}
@@ -767,7 +817,7 @@ func (svc *Service) GetAlerts(ctx context.Context, fp *services.FilterParams) ([
 }
 
 // FindAlertsByID searches alerts by IDs in alertmanager.
-func (svc *Service) FindAlertsByID(ctx context.Context, params *services.FilterParams, ids []string) ([]*ammodels.GettableAlert, error) {
+func (svc *Service) FindAlertsByID(ctx context.Context, params *services.FilterParams, ids []string) ([]*gmodels.GettableAlert, error) {
 	alerts, err := svc.GetAlerts(ctx, params)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to get alerts form alertmanager")
@@ -779,7 +829,7 @@ func (svc *Service) FindAlertsByID(ctx context.Context, params *services.FilterP
 		m[id] = struct{}{}
 	}
 
-	res := make([]*ammodels.GettableAlert, 0, l)
+	res := make([]*gmodels.GettableAlert, 0, l)
 	for _, a := range alerts {
 		if _, ok := m[*a.Fingerprint]; ok {
 			res = append(res, a)
@@ -790,7 +840,7 @@ func (svc *Service) FindAlertsByID(ctx context.Context, params *services.FilterP
 }
 
 // SilenceAlerts silences a group of provided alerts.
-func (svc *Service) SilenceAlerts(ctx context.Context, alerts []*ammodels.GettableAlert) error {
+func (svc *Service) SilenceAlerts(ctx context.Context, alerts []*gmodels.GettableAlert) error {
 	var err error
 	for _, a := range alerts {
 		if len(a.Status.SilencedBy) != 0 {
@@ -804,7 +854,7 @@ func (svc *Service) SilenceAlerts(ctx context.Context, alerts []*ammodels.Gettab
 				&ammodels.Matcher{
 					IsRegex: pointer.ToBool(false),
 					Name:    pointer.ToString(label),
-					Value:   pointer.ToString(value),
+					Value:   pointer.ToString(string(value)),
 				})
 		}
 
@@ -831,7 +881,7 @@ func (svc *Service) SilenceAlerts(ctx context.Context, alerts []*ammodels.Gettab
 }
 
 // UnsilenceAlerts unmutes the provided alerts.
-func (svc *Service) UnsilenceAlerts(ctx context.Context, alerts []*ammodels.GettableAlert) error {
+func (svc *Service) UnsilenceAlerts(ctx context.Context, alerts []*gmodels.GettableAlert) error {
 	var err error
 	for _, a := range alerts {
 		for _, silenceID := range a.Status.SilencedBy {
@@ -877,5 +927,5 @@ func (svc *Service) IsReady(ctx context.Context) error {
 // configure default client; we use it mainly because we can't remove it from generated code
 //nolint:gochecknoinits
 func init() {
-	amclient.Default.SetTransport(httptransport.New("127.0.0.1:9093", "/alertmanager/api/v2", []string{"http"}))
+	gclient.Default.SetTransport(httptransport.New("127.0.0.1:3000", "/", []string{"http"}))
 }

services/checks/checks.go

@@ -37,6 +37,7 @@ import (
 	"github.com/percona-platform/saas/pkg/check"
 	"github.com/percona-platform/saas/pkg/common"
 	"github.com/percona/pmm/api/agentpb"
+	"github.com/percona/pmm/api/grafana/gmodels"
 	"github.com/percona/pmm/utils/pdeathsig"
 	"github.com/percona/pmm/version"
 	"github.com/pkg/errors"
@@ -297,6 +298,14 @@ func (s *Service) GetSecurityCheckResults() ([]services.CheckResult, error) {
 	return s.alertsRegistry.getCheckResults(), nil
 }
 
+func labelSetToMap(set gmodels.LabelSet) map[string]string {
+	res := make(map[string]string, len(set))
+	for key, value := range set {
+		res[key] = string(value)
+	}
+	return res
+}
+
 // GetChecksResults returns the failed checks for a given service from AlertManager.
 func (s *Service) GetChecksResults(ctx context.Context, serviceID string) ([]services.CheckResult, error) {
 	settings, err := models.GetSettings(s.db)
@@ -320,23 +329,23 @@ func (s *Service) GetChecksResults(ctx context.Context, serviceID string) ([]ser
 	checkResults := make([]services.CheckResult, 0, len(res))
 	for _, alert := range res {
 		checkResults = append(checkResults, services.CheckResult{
-			CheckName: alert.Labels[model.AlertNameLabel],
+			CheckName: string(alert.Labels[model.AlertNameLabel]),
 			Silenced:  len(alert.Status.SilencedBy) != 0,
-			AlertID:   alert.Labels["alert_id"],
+			AlertID:   string(alert.Labels["alert_id"]),
 			Interval:  check.Interval(alert.Labels["interval_group"]),
 			Target: services.Target{
-				AgentID:     alert.Labels["agent_id"],
-				ServiceID:   alert.Labels["service_id"],
-				ServiceName: alert.Labels["service_name"],
-				NodeName:    alert.Labels["node_name"],
-				Labels:      alert.Labels,
+				AgentID:     string(alert.Labels["agent_id"]),
+				ServiceID:   string(alert.Labels["service_id"]),
+				ServiceName: string(alert.Labels["service_name"]),
+				NodeName:    string(alert.Labels["node_name"]),
+				Labels:      labelSetToMap(alert.Labels),
 			},
 			Result: check.Result{
-				Summary:     alert.Annotations["summary"],
-				Description: alert.Annotations["description"],
-				ReadMoreURL: alert.Annotations["read_more_url"],
-				Severity:    common.ParseSeverity(alert.Labels["severity"]),
-				Labels:      alert.Labels,
+				Summary:     string(alert.Annotations["summary"]),
+				Description: string(alert.Annotations["description"]),
+				ReadMoreURL: string(alert.Annotations["read_more_url"]),
+				Severity:    common.ParseSeverity(string(alert.Labels["severity"])),
+				Labels:      labelSetToMap(alert.Labels),
 			},
 		})
 	}

services/checks/deps.go

@@ -18,6 +18,7 @@ package checks
 
 import (
 	"context"
+	"github.com/percona/pmm/api/grafana/gmodels"
 
 	"github.com/percona/pmm/api/alertmanager/ammodels"
 
@@ -46,7 +47,7 @@ type agentsRegistry interface {
 // We use it instead of real type for testing and to avoid dependency cycle.
 type alertmanagerService interface {
 	SendAlerts(ctx context.Context, alerts ammodels.PostableAlerts)
-	SilenceAlerts(ctx context.Context, alerts []*ammodels.GettableAlert) error
-	UnsilenceAlerts(ctx context.Context, alerts []*ammodels.GettableAlert) error
-	GetAlerts(ctx context.Context, params *services.FilterParams) ([]*ammodels.GettableAlert, error)
+	SilenceAlerts(ctx context.Context, alerts []*gmodels.GettableAlert) error
+	UnsilenceAlerts(ctx context.Context, alerts []*gmodels.GettableAlert) error
+	GetAlerts(ctx context.Context, params *services.FilterParams) ([]*gmodels.GettableAlert, error)
 }

services/management/ia/alerts_service.go

@@ -24,7 +24,7 @@ import (
 	"time"
 
 	"github.com/percona-platform/saas/pkg/common"
-	"github.com/percona/pmm/api/alertmanager/ammodels"
+	"github.com/percona/pmm/api/grafana/gmodels"
 	"github.com/percona/pmm/api/managementpb"
 	iav1beta1 "github.com/percona/pmm/api/managementpb/ia"
 	"github.com/pkg/errors"
@@ -68,14 +68,22 @@ func (s *AlertsService) Enabled() bool {
 	return settings.IntegratedAlerting.Enabled
 }
 
+func labelSetToMap(set gmodels.LabelSet) map[string]string {
+	res := make(map[string]string, len(set))
+	for key, value := range set {
+		res[key] = string(value)
+	}
+	return res
+}
+
 // ListAlerts returns list of existing alerts.
 func (s *AlertsService) ListAlerts(ctx context.Context, req *iav1beta1.ListAlertsRequest) (*iav1beta1.ListAlertsResponse, error) {
 	filter := &services.FilterParams{
-		IsIA: true,
+		ExcludeChecks: true,
 	}
 	alerts, err := s.alertManager.GetAlerts(ctx, filter)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to get alerts form alertmanager")
+		return nil, errors.Wrap(err, "failed to get alerts from alertmanager")
 	}
 
 	var res []*iav1beta1.Alert
@@ -99,16 +107,16 @@ func (s *AlertsService) ListAlerts(ctx context.Context, req *iav1beta1.ListAlert
 			st = iav1beta1.Status_SILENCED
 		}
 
-		var rule *iav1beta1.Rule
+		rule := &iav1beta1.Rule{}
 		// Rules files created by user in directory /srv/prometheus/rules/ doesn't have associated rules in DB.
 		// So alertname field will be empty or will keep invalid value. Don't fill rule field in that case.
 		ruleID, ok := alert.Labels["alertname"]
-		if ok && strings.HasPrefix(ruleID, "/rule_id/") {
+		if ok && strings.HasPrefix(string(ruleID), "/rule_id/") {
 			var r *models.Rule
 			var channels []*models.Channel
 			e := s.db.InTransaction(func(tx *reform.TX) error {
 				var err error
-				r, err = models.FindRuleByID(tx.Querier, ruleID)
+				r, err = models.FindRuleByID(tx.Querier, string(ruleID))
 				if err != nil {
 					return err
 				}
@@ -132,6 +140,7 @@ func (s *AlertsService) ListAlerts(ctx context.Context, req *iav1beta1.ListAlert
 				return nil, errors.Wrapf(err, "failed to convert alert rule")
 			}
 		}
+
 		pass, err := satisfiesFilters(alert, rule.Filters)
 		if err != nil {
 			return nil, err
@@ -143,10 +152,10 @@ func (s *AlertsService) ListAlerts(ctx context.Context, req *iav1beta1.ListAlert
 
 		res = append(res, &iav1beta1.Alert{
 			AlertId:   getAlertID(alert),
-			Summary:   alert.Annotations["summary"],
-			Severity:  managementpb.Severity(common.ParseSeverity(alert.Labels["severity"])),
+			Summary:   string(alert.Annotations["summary"]),
+			Severity:  managementpb.Severity(common.ParseSeverity(string(alert.Labels["severity"]))),
 			Status:    st,
-			Labels:    alert.Labels,
+			Labels:    labelSetToMap(alert.Labels),
 			Rule:      rule,
 			CreatedAt: createdAt,
 			UpdatedAt: updatedAt,
@@ -185,7 +194,7 @@ func (s *AlertsService) ListAlerts(ctx context.Context, req *iav1beta1.ListAlert
 }
 
 // satisfiesFilters checks that alert passes filters, returns true in case of success.
-func satisfiesFilters(alert *ammodels.GettableAlert, filters []*iav1beta1.Filter) (bool, error) {
+func satisfiesFilters(alert *gmodels.GettableAlert, filters []*iav1beta1.Filter) (bool, error) {
 	for _, filter := range filters {
 		value, ok := alert.Labels[filter.Key]
 		if !ok {
@@ -194,7 +203,7 @@ func satisfiesFilters(alert *ammodels.GettableAlert, filters []*iav1beta1.Filter
 
 		switch filter.Type {
 		case iav1beta1.FilterType_EQUAL:
-			if filter.Value != value {
+			if filter.Value != string(value) {
 				return false, nil
 			}
 		case iav1beta1.FilterType_REGEX:
@@ -216,14 +225,14 @@ func satisfiesFilters(alert *ammodels.GettableAlert, filters []*iav1beta1.Filter
 	return true, nil
 }
 
-func getAlertID(alert *ammodels.GettableAlert) string {
+func getAlertID(alert *gmodels.GettableAlert) string {
 	return *alert.Fingerprint
 }
 
 // ToggleAlerts allows to silence/unsilence specified alerts.
 func (s *AlertsService) ToggleAlerts(ctx context.Context, req *iav1beta1.ToggleAlertsRequest) (*iav1beta1.ToggleAlertsResponse, error) {
 	var err error
-	var alerts []*ammodels.GettableAlert
+	var alerts []*gmodels.GettableAlert
 
 	filters := &services.FilterParams{
 		IsIA: true,