PMM-5492 Add Pprof data to logs.zip

go.mod

@@ -36,7 +36,7 @@ require (
 	github.com/minio/minio-go/v7 v7.0.26
 	github.com/percona-platform/dbaas-api v0.0.0-20220110092915-5aacd784d472
 	github.com/percona-platform/saas v0.0.0-20220427162947-f9d246ad0f16
-	github.com/percona/pmm v0.0.0-20220520150831-23069cdf1bb8
+	github.com/percona/pmm v0.0.0-20220530213957-dc4793335a14
 	github.com/percona/promconfig v0.2.4-0.20211110115058-98687f586f54
 	github.com/pkg/errors v0.9.1
 	github.com/pmezard/go-difflib v1.0.0

go.sum

@@ -470,6 +470,8 @@ github.com/percona-platform/saas v0.0.0-20220427162947-f9d246ad0f16 h1:0fx16uGtl
 github.com/percona-platform/saas v0.0.0-20220427162947-f9d246ad0f16/go.mod h1:gFUwaFp6Ugu5qsBwiOVJYbDlzgZ77tmXdXGO7tG5xVI=
 github.com/percona/pmm v0.0.0-20220520150831-23069cdf1bb8 h1:P5iuV4GRUIviRg/5/FM6ZOKdiBPdwUPbrHld/epM3hk=
 github.com/percona/pmm v0.0.0-20220520150831-23069cdf1bb8/go.mod h1:gr+WLd8clEAe2xMFgsGhpw9ziZc2UCWcfy6d3M6Aq00=
+github.com/percona/pmm v0.0.0-20220530213957-dc4793335a14 h1:NvhRMv1rftqvSpzmFHQlmSPFIjsUxv7Ii4anCnV46Dk=
+github.com/percona/pmm v0.0.0-20220530213957-dc4793335a14/go.mod h1:gr+WLd8clEAe2xMFgsGhpw9ziZc2UCWcfy6d3M6Aq00=
 github.com/percona/promconfig v0.2.4-0.20211110115058-98687f586f54 h1:aI1emmycDTGWKsBdxFPKZqohfBbK4y2ta9G4+RX7gVg=
 github.com/percona/promconfig v0.2.4-0.20211110115058-98687f586f54/go.mod h1:Y2uXi5QNk71+ceJHuI9poank+0S1kjxd3K105fXKVkg=
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=

main.go

@@ -93,6 +93,7 @@ import (
 	"github.com/percona/pmm-managed/utils/clean"
 	"github.com/percona/pmm-managed/utils/interceptors"
 	"github.com/percona/pmm-managed/utils/logger"
+	"github.com/percona/pmm-managed/utils/pprof"
 )
 
 const (
@@ -110,9 +111,23 @@ const (
 func addLogsHandler(mux *http.ServeMux, logs *supervisord.Logs) {
 	l := logrus.WithField("component", "logs.zip")
 
+	cfg := config.NewService()
+	if err := cfg.Load(); err != nil {
+		l.Panicf("Failed to load config: %+v", err)
+	}
+
 	mux.HandleFunc("/logs.zip", func(rw http.ResponseWriter, req *http.Request) {
+		contextTimeout := 10 * time.Second
+		// increase context timeout if pprof query parameter exist in request
+		pprofQueryParameter, _ := strconv.Atoi(req.FormValue("pprof"))
+		var pprofSettings *pprof.Config
+		if pprofQueryParameter > 0 {
+			contextTimeout += cfg.Config.Services.Pprof.ProfileDuration + cfg.Config.Services.Pprof.TraceDuration
+			pprofSettings = &cfg.Config.Services.Pprof
+		}
+
 		// fail-safe
-		ctx, cancel := context.WithTimeout(req.Context(), 10*time.Second)
+		ctx, cancel := context.WithTimeout(req.Context(), contextTimeout)
 		defer cancel()
 
 		filename := fmt.Sprintf("pmm-server_%s.zip", time.Now().UTC().Format("2006-01-02_15-04"))
@@ -121,7 +136,7 @@ func addLogsHandler(mux *http.ServeMux, logs *supervisord.Logs) {
 		rw.Header().Set(`Content-Disposition`, `attachment; filename="`+filename+`"`)
 
 		ctx = logger.Set(ctx, "logs")
-		if err := logs.Zip(ctx, rw); err != nil {
+		if err := logs.Zip(ctx, rw, pprofSettings); err != nil {
 			l.Errorf("%+v", err)
 		}
 	})

services/config/config.go

@@ -28,6 +28,7 @@ import (
 
 	"github.com/percona/pmm-managed/services/platform"
 	"github.com/percona/pmm-managed/services/telemetry"
+	"github.com/percona/pmm-managed/utils/pprof"
 )
 
 const (
@@ -49,6 +50,7 @@ type Config struct {
 	Services struct {
 		Platform  platform.Config         `yaml:"platform"`
 		Telemetry telemetry.ServiceConfig `yaml:"telemetry"`
+		Pprof     pprof.Config            `yaml:"pprof"`
 	} `yaml:"services"`
 }
 
@@ -94,7 +96,7 @@ func (s *Service) Load() error {
 	if err := cfg.Services.Telemetry.Init(s.l); err != nil {
 		return err
 	}
-
+	cfg.Services.Pprof.Init()
 	s.Config = cfg
 
 	return nil

services/config/pmm-managed.yaml

@@ -32,3 +32,6 @@ services:
       retry_backoff_env: "PERCONA_TEST_TELEMETRY_RETRY_BACKOFF"
       retry_count: 20
       send_timeout: 5s
+  pprof:
+    profile_duration: 30s
+    trace_duration: 10s

services/supervisord/logs.go

@@ -38,6 +38,7 @@ import (
 	"golang.org/x/sys/unix"
 
 	"github.com/percona/pmm-managed/utils/logger"
+	pprofUtils "github.com/percona/pmm-managed/utils/pprof"
 )
 
 const (
@@ -69,7 +70,7 @@ func NewLogs(pmmVersion string, pmmUpdateChecker *PMMUpdateChecker) *Logs {
 }
 
 // Zip creates .zip archive with all logs.
-func (l *Logs) Zip(ctx context.Context, w io.Writer) error {
+func (l *Logs) Zip(ctx context.Context, w io.Writer, pprofSettings *pprofUtils.Config) error {
 	start := time.Now()
 	log := logger.Get(ctx).WithField("component", "logs")
 	log.WithField("d", time.Since(start).Seconds()).Info("Starting...")
@@ -80,7 +81,7 @@ func (l *Logs) Zip(ctx context.Context, w io.Writer) error {
 	zw := zip.NewWriter(w)
 	now := time.Now().UTC()
 
-	files := l.files(ctx)
+	files := l.files(ctx, pprofSettings)
 	log.WithField("d", time.Since(start).Seconds()).Infof("Collected %d files.", len(files))
 
 	for _, file := range files {
@@ -127,8 +128,8 @@ func (l *Logs) Zip(ctx context.Context, w io.Writer) error {
 	return nil
 }
 
-// files reads log/config files and returns content.
-func (l *Logs) files(ctx context.Context) []fileContent {
+// files reads log/config/pprof files and returns content.
+func (l *Logs) files(ctx context.Context, pprofSettings *pprofUtils.Config) []fileContent {
 	files := make([]fileContent, 0, 20)
 
 	// add logs
@@ -214,6 +215,30 @@ func (l *Logs) files(ctx context.Context) []fileContent {
 		Err:  err,
 	})
 
+	// add pprof
+	if pprofSettings != nil {
+		traceBytes, err := pprofUtils.Trace(pprofSettings.TraceDuration)
+		files = append(files, fileContent{
+			Name: "pprof/trace.out",
+			Data: traceBytes,
+			Err:  err,
+		})
+
+		profileBytes, err := pprofUtils.Profile(pprofSettings.ProfileDuration)
+		files = append(files, fileContent{
+			Name: "pprof/profile.pb.gz",
+			Data: profileBytes,
+			Err:  err,
+		})
+
+		heapBytes, err := pprofUtils.Heap(true)
+		files = append(files, fileContent{
+			Name: "pprof/heap.pb.gz",
+			Data: heapBytes,
+			Err:  err,
+		})
+	}
+
 	sort.Slice(files, func(i, j int) bool { return files[i].Name < files[j].Name })
 	return files
 }

services/supervisord/logs_test.go

@@ -34,6 +34,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/percona/pmm-managed/utils/logger"
+	"github.com/percona/pmm-managed/utils/pprof"
 )
 
 var commonExpectedFiles = []string{
@@ -55,6 +56,9 @@ var commonExpectedFiles = []string{
 	"pmm.conf",
 	"pmm.ini",
 	"postgresql.log",
+	"pprof/heap.pb.gz",
+	"pprof/profile.pb.gz",
+	"pprof/trace.out",
 	"qan-api2.ini",
 	"qan-api2.log",
 	"supervisorctl_status.log",
@@ -124,7 +128,10 @@ func TestFiles(t *testing.T) {
 	l := NewLogs("2.4.5", checker)
 	ctx := logger.Set(context.Background(), t.Name())
 
-	files := l.files(ctx)
+	files := l.files(ctx, &pprof.Config{
+		ProfileDuration: 1 * time.Second,
+		TraceDuration:   1 * time.Second,
+	})
 	actual := make([]string, 0, len(files))
 	for _, f := range files {
 		// present only after update
@@ -157,7 +164,10 @@ func TestZip(t *testing.T) {
 	ctx := logger.Set(context.Background(), t.Name())
 
 	var buf bytes.Buffer
-	require.NoError(t, l.Zip(ctx, &buf))
+	require.NoError(t, l.Zip(ctx, &buf, &pprof.Config{
+		ProfileDuration: 1 * time.Second,
+		TraceDuration:   1 * time.Second,
+	}))
 	reader := bytes.NewReader(buf.Bytes())
 	r, err := zip.NewReader(reader, reader.Size())
 	require.NoError(t, err)

utils/pprof/pprof.go

@@ -0,0 +1,79 @@
+// pmm-managed
+// Copyright (C) 2017 Percona LLC
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+package pprof
+
+import (
+	"bytes"
+	"fmt"
+	"runtime"
+	"runtime/pprof"
+	"runtime/trace"
+	"time"
+)
+
+// Profile responds with the pprof-formatted cpu profile.
+// Profiling lasts for duration specified in seconds.
+func Profile(duration time.Duration) ([]byte, error) {
+	var profileBuf bytes.Buffer
+	if err := pprof.StartCPUProfile(&profileBuf); err != nil {
+		return nil, err
+	}
+
+	time.Sleep(duration)
+	pprof.StopCPUProfile()
+
+	return profileBuf.Bytes(), nil
+}
+
+// Trace responds with the execution trace in binary form.
+// Tracing lasts for duration specified in seconds.
+func Trace(duration time.Duration) ([]byte, error) {
+	var traceBuf bytes.Buffer
+	if err := trace.Start(&traceBuf); err != nil {
+		return nil, err
+	}
+
+	time.Sleep(duration)
+	trace.Stop()
+
+	return traceBuf.Bytes(), nil
+}
+
+// Heap responds with the pprof-formatted profile named "heap".
+// listing the available profiles.
+// You can specify the gc parameter to run gc before taking the heap sample.
+func Heap(gc bool) ([]byte, error) {
+	var heapBuf bytes.Buffer
+	debug := 0
+	profile := "heap"
+
+	p := pprof.Lookup(profile)
+	if p == nil {
+		return nil, fmt.Errorf("profile cannot be found: %s", profile)
+	}
+
+	if gc {
+		runtime.GC()
+	}
+
+	err := p.WriteTo(&heapBuf, debug)
+	if err != nil {
+		return nil, err
+	}
+
+	return heapBuf.Bytes(), nil
+}

utils/pprof/pprof_config.go

@@ -0,0 +1,31 @@
+// pmm-managed
+// Copyright (C) 2017 Percona LLC
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+package pprof
+
+import (
+	"time"
+)
+
+// Config pprof settings.
+type Config struct {
+	ProfileDuration time.Duration `yaml:"profile_duration"` //nolint:tagliatelle
+	TraceDuration   time.Duration `yaml:"trace_duration"`   //nolint:tagliatelle
+}
+
+// Init pprof config init.
+func (c *Config) Init() {
+}

utils/pprof/pprof_test.go

@@ -0,0 +1,72 @@
+// pmm-managed
+// Copyright (C) 2017 Percona LLC
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+package pprof
+
+import (
+	"bytes"
+	"compress/gzip"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestHeap(t *testing.T) {
+	t.Parallel()
+	t.Run("Heap test", func(t *testing.T) {
+		heapBytes, err := Heap(true)
+
+		// read gzip
+		reader, err := gzip.NewReader(bytes.NewBuffer(heapBytes))
+		assert.NoError(t, err)
+
+		var resB bytes.Buffer
+		_, err = resB.ReadFrom(reader)
+		assert.NoError(t, err)
+		assert.True(t, len(resB.Bytes()) != 0)
+	})
+}
+
+func TestProfile(t *testing.T) {
+	t.Parallel()
+	t.Run("Profile test", func(t *testing.T) {
+		profileBytes, err := Profile(1 * time.Second)
+
+		assert.NoError(t, err)
+		assert.True(t, len(profileBytes) != 0)
+
+		// read gzip
+		reader, err := gzip.NewReader(bytes.NewBuffer(profileBytes))
+		assert.NoError(t, err)
+
+		var resB bytes.Buffer
+		_, err = resB.ReadFrom(reader)
+		assert.NoError(t, err)
+
+		assert.True(t, len(resB.Bytes()) != 0)
+	})
+}
+
+func TestTrace(t *testing.T) {
+	t.Parallel()
+	t.Run("Trace test", func(t *testing.T) {
+		traceBytes, err := Trace(1 * time.Second)
+
+		assert.NoError(t, err)
+		assert.True(t, len(traceBytes) != 0)
+	})
+}