PMM-9377 upgrade to grpc-gateway v2

api-tests/helpers.go

@@ -70,12 +70,14 @@ func AssertAPIErrorf(t TestingT, actual error, httpStatus int, grpcCode codes.Co
 	require.True(t, codeField.IsValid(), "Wrong response structure. There is no field Code in Payload.")
 	assert.Equal(t, int64(grpcCode), codeField.Int(), "gRPC status codes are not equal")
 
-	errorField := payload.Elem().FieldByName("Error")
-	require.True(t, errorField.IsValid(), "Wrong response structure. There is no field Error in Payload.")
+	errorField := payload.Elem().FieldByName("Message")
+	require.True(t, errorField.IsValid(), "Wrong response structure. There is no field Message in Payload.")
 	if len(a) != 0 {
 		format = fmt.Sprintf(format, a...)
 	}
-	assert.Equal(t, format, errorField.String())
+	// We use "assert.Contains" because some error messages include info that changes easily
+	//(e.g. the line number in the proto file).
+	assert.Contains(t, errorField.String(), format)
 }
 
 func ExpectFailure(t *testing.T, link string) *expectedFailureTestingT {

api-tests/server/auth_test.go

@@ -131,7 +131,7 @@ func TestSetup(t *testing.T) {
 			"swagger/":    301,
 
 			"v1/readyz":           200,
-			"v1/AWSInstanceCheck": 405, // only POST is expected
+			"v1/AWSInstanceCheck": 501, // only POST is expected, other request methods are seen as unimplemented
 			"v1/version":          401, // Grafana authentication required
 		}
 		for path, code := range paths {
@@ -173,7 +173,7 @@ func TestSetup(t *testing.T) {
 
 		resp, b := doRequest(t, client, req)
 		assert.Equal(t, 200, resp.StatusCode, "response:\n%s", b)
-		assert.Equal(t, "{\n\n}", string(b), "response:\n%s", b)
+		assert.Equal(t, "{}", string(b), "response:\n%s", b)
 	})
 }
 

api-tests/server/checks_test.go

@@ -162,7 +162,7 @@ func TestChangeSecurityChecks(t *testing.T) {
 			}
 
 			_, err = managementClient.Default.SecurityChecks.ChangeSecurityChecks(params)
-			pmmapitests.AssertAPIErrorf(t, err, 400, codes.InvalidArgument, "unknown value \"\\\"unknown_interval\\\"\" for enum management.SecurityCheckInterval")
+			pmmapitests.AssertAPIErrorf(t, err, 400, codes.InvalidArgument, "invalid value for enum type: \"unknown_interval\"")
 
 			resp, err = managementClient.Default.SecurityChecks.ListSecurityChecks(nil)
 			require.NoError(t, err)

api-tests/server/readyz_test.go

@@ -54,7 +54,7 @@ func TestReadyz(t *testing.T) {
 			b, err := ioutil.ReadAll(resp.Body)
 			require.NoError(t, err)
 			assert.Equal(t, 200, resp.StatusCode, "response:\n%s", b)
-			assert.Equal(t, "{\n\n}", string(b))
+			assert.Equal(t, "{}", string(b))
 		})
 	}
 }

api-tests/server/settings_test.go

@@ -557,7 +557,7 @@ func TestSettings(t *testing.T) {
 					Context: pmmapitests.Context,
 				})
 				pmmapitests.AssertAPIErrorf(t, err, 400, codes.InvalidArgument,
-					`bad Duration: time: missing unit in duration "1"`)
+					`invalid google.protobuf.Duration value "1"`)
 				assert.Empty(t, res)
 			})
 
@@ -605,7 +605,7 @@ func TestSettings(t *testing.T) {
 					Context: pmmapitests.Context,
 				})
 				pmmapitests.AssertAPIErrorf(t, err, 400, codes.InvalidArgument,
-					`bad Duration: time: missing unit in duration "1"`)
+					`invalid google.protobuf.Duration value "1"`)
 				assert.Empty(t, res)
 			})
 
@@ -651,7 +651,7 @@ func TestSettings(t *testing.T) {
 					Context: pmmapitests.Context,
 				})
 				pmmapitests.AssertAPIErrorf(t, err, 400, codes.InvalidArgument,
-					`bad Duration: time: missing unit in duration "1"`)
+					`invalid google.protobuf.Duration value "1"`)
 				assert.Empty(t, res)
 			})
 
@@ -679,7 +679,7 @@ func TestSettings(t *testing.T) {
 					Context: pmmapitests.Context,
 				})
 				pmmapitests.AssertAPIErrorf(t, err, 400, codes.InvalidArgument,
-					`Invalid argument: data_retention: should be a natural number of days.`)
+					`invalid google.protobuf.Duration value "36h"`)
 				assert.Empty(t, res)
 			})
 
@@ -723,9 +723,9 @@ func TestSettings(t *testing.T) {
 						MetricsResolutions: &server.ChangeSettingsParamsBodyMetricsResolutions{
 							Hr: "2s",
 							Mr: "15s",
-							Lr: "2m",
+							Lr: "120s", // 2 minutes
 						},
-						DataRetention: "240h",
+						DataRetention: "864000s",                           // 240 hours
 						AWSPartitions: []string{"aws-cn", "aws", "aws-cn"}, // duplicates are ok
 					},
 					Context: pmmapitests.Context,
@@ -790,8 +790,8 @@ func TestSettings(t *testing.T) {
 				res, err := serverClient.Default.Server.ChangeSettings(&server.ChangeSettingsParams{
 					Body: server.ChangeSettingsBody{
 						SttCheckIntervals: &server.ChangeSettingsParamsBodySttCheckIntervals{
-							RareInterval:     "8h",
-							StandardInterval: "30m",
+							RareInterval:     "28800s", // 8 hours
+							StandardInterval: "1800s",  // 30 minutes
 							FrequentInterval: "20s",
 						},
 					},
@@ -956,8 +956,8 @@ groups:
 					"60s": "60s",
 					"61s": "61s",
 					"61":  "", // no suffix => error
-					"2m":  "120s",
-					"1h":  "3600s",
+					"2m":  "", // m suffix => error
+					"1h":  "", // h suffix => error
 					"1d":  "", // d suffix => error
 					"1w":  "", // w suffix => error
 				} {

go.mod

@@ -30,13 +30,13 @@ require (
 	github.com/google/uuid v1.3.0
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
-	github.com/grpc-ecosystem/grpc-gateway v1.16.0
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.10.0
 	github.com/hashicorp/go-version v1.4.0
 	github.com/lib/pq v1.10.6
 	github.com/minio/minio-go/v7 v7.0.27
 	github.com/percona-platform/dbaas-api v0.0.0-20220110092915-5aacd784d472
 	github.com/percona-platform/saas v0.0.0-20220427162947-f9d246ad0f16
-	github.com/percona/pmm v0.0.0-20220520150831-23069cdf1bb8
+	github.com/percona/pmm v0.0.0-20220601161301-c794e78371a7
 	github.com/percona/promconfig v0.2.4-0.20211110115058-98687f586f54
 	github.com/pkg/errors v0.9.1
 	github.com/pmezard/go-difflib v1.0.0
@@ -89,7 +89,6 @@ require (
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v4 v4.3.0 // indirect
 	github.com/google/btree v1.0.1 // indirect
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.10.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
 	github.com/hashicorp/go-msgpack v1.1.5 // indirect
@@ -128,8 +127,8 @@ require (
 	github.com/shopspring/decimal v1.3.1 // indirect
 	github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 // indirect
 	github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546 // indirect
-	github.com/stretchr/objx v0.3.0 // indirect
-	go.mongodb.org/mongo-driver v1.9.0 // indirect
+	github.com/stretchr/objx v0.4.0 // indirect
+	go.mongodb.org/mongo-driver v1.9.1 // indirect
 	go.opentelemetry.io/otel v1.7.0 // indirect
 	go.opentelemetry.io/otel/trace v1.7.0 // indirect
 	golang.org/x/mod v0.5.1 // indirect

go.sum

@@ -311,7 +311,6 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaW
 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
-github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.10.0 h1:ESEyqQqXXFIcImj/BE8oKEX37Zsuceb2cZI+EL/zNCY=
 github.com/grpc-ecosystem/grpc-gateway/v2 v2.10.0/go.mod h1:XnLCLFp3tjoZJszVKjfpyAK6J8sYIcQXWQxmqLWF21I=
@@ -468,8 +467,8 @@ github.com/percona-platform/dbaas-api v0.0.0-20220110092915-5aacd784d472 h1:Henk
 github.com/percona-platform/dbaas-api v0.0.0-20220110092915-5aacd784d472/go.mod h1:WZZ3Hi+lAWCaGWmsrfkkvRQPkIa8n1OZ0s8Su+vbgus=
 github.com/percona-platform/saas v0.0.0-20220427162947-f9d246ad0f16 h1:0fx16uGtl4MwrBwm9/VSoNEhjL0cXYxS0quEhLthGcc=
 github.com/percona-platform/saas v0.0.0-20220427162947-f9d246ad0f16/go.mod h1:gFUwaFp6Ugu5qsBwiOVJYbDlzgZ77tmXdXGO7tG5xVI=
-github.com/percona/pmm v0.0.0-20220520150831-23069cdf1bb8 h1:P5iuV4GRUIviRg/5/FM6ZOKdiBPdwUPbrHld/epM3hk=
-github.com/percona/pmm v0.0.0-20220520150831-23069cdf1bb8/go.mod h1:gr+WLd8clEAe2xMFgsGhpw9ziZc2UCWcfy6d3M6Aq00=
+github.com/percona/pmm v0.0.0-20220601161301-c794e78371a7 h1:0F1oQzLtLaL/pyySckUgHrDV8bFpLHDqlp+cI+JGgOM=
+github.com/percona/pmm v0.0.0-20220601161301-c794e78371a7/go.mod h1:o6L/1dDmEPeIOaQkWQAChYPgl33xWdgDyCkDoweSQsA=
 github.com/percona/promconfig v0.2.4-0.20211110115058-98687f586f54 h1:aI1emmycDTGWKsBdxFPKZqohfBbK4y2ta9G4+RX7gVg=
 github.com/percona/promconfig v0.2.4-0.20211110115058-98687f586f54/go.mod h1:Y2uXi5QNk71+ceJHuI9poank+0S1kjxd3K105fXKVkg=
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
@@ -551,8 +550,8 @@ github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As=
-github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -580,8 +579,8 @@ github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
 go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg=
 go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng=
 go.mongodb.org/mongo-driver v1.8.3/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY=
-go.mongodb.org/mongo-driver v1.9.0 h1:f3aLGJvQmBl8d9S40IL+jEyBC6hfLPbJjv9t5hEM9ck=
-go.mongodb.org/mongo-driver v1.9.0/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY=
+go.mongodb.org/mongo-driver v1.9.1 h1:m078y9v7sBItkt1aaoe2YlvWEXcD263e1a4E1fBrJ1c=
+go.mongodb.org/mongo-driver v1.9.1/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=

main.go

@@ -39,7 +39,7 @@ import (
 	grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
 	grpc_validator "github.com/grpc-ecosystem/go-grpc-middleware/validator"
 	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
-	grpc_gateway "github.com/grpc-ecosystem/grpc-gateway/runtime"
+	grpc_gateway "github.com/grpc-ecosystem/grpc-gateway/v2/runtime"
 	"github.com/percona/pmm/api/agentpb"
 	"github.com/percona/pmm/api/inventorypb"
 	"github.com/percona/pmm/api/managementpb"
@@ -49,6 +49,7 @@ import (
 	iav1beta1 "github.com/percona/pmm/api/managementpb/ia"
 	"github.com/percona/pmm/api/platformpb"
 	"github.com/percona/pmm/api/serverpb"
+	pmmerrors "github.com/percona/pmm/utils/errors"
 	"github.com/percona/pmm/utils/sqlmetrics"
 	"github.com/percona/pmm/version"
 	prom "github.com/prometheus/client_golang/prometheus"
@@ -60,6 +61,7 @@ import (
 	"google.golang.org/grpc/credentials/insecure"
 	"google.golang.org/grpc/grpclog"
 	"google.golang.org/grpc/reflection"
+	"google.golang.org/protobuf/encoding/protojson"
 	"gopkg.in/alecthomas/kingpin.v2"
 	"gopkg.in/reform.v1"
 	"gopkg.in/reform.v1/dialects/postgresql"
@@ -282,20 +284,26 @@ func runHTTP1Server(ctx context.Context, deps *http1ServerDeps) {
 	l.Infof("Starting server on http://%s/ ...", http1Addr)
 
 	marshaller := &grpc_gateway.JSONPb{
-		OrigName:     true,
-		EnumsAsInts:  false,
-		EmitDefaults: false,
-		Indent:       "  ",
+		MarshalOptions: protojson.MarshalOptions{
+			UseEnumNumbers:  false,
+			EmitUnpopulated: false,
+			UseProtoNames:   true,
+			Indent:          "  ",
+		},
+		UnmarshalOptions: protojson.UnmarshalOptions{
+			DiscardUnknown: true,
+		},
 	}
 
 	// FIXME make that a default behavior: https://jira.percona.com/browse/PMM-6722
 	if nicer, _ := strconv.ParseBool(os.Getenv("PERCONA_TEST_NICER_API")); nicer {
 		l.Warn("Enabling nicer API with default/zero values in response.")
-		marshaller.EmitDefaults = true
+		marshaller.EmitUnpopulated = true
 	}
 
 	proxyMux := grpc_gateway.NewServeMux(
 		grpc_gateway.WithMarshalerOption(grpc_gateway.MIMEWildcard, marshaller),
+		grpc_gateway.WithErrorHandler(pmmerrors.PMMHTTPErrorHandler),
 	)
 	opts := []grpc.DialOption{
 		grpc.WithTransportCredentials(insecure.NewCredentials()),