WIP [dbnode] Refactor posting list operations to be lazily evaluated and zero copy

src/cmd/services/m3coordinator/ingest/carbon/ingest.go

@@ -126,15 +126,21 @@ func NewIngester(
 		}
 	})
 
+	scope := opts.InstrumentOptions.MetricsScope()
+	metrics, err := newCarbonIngesterMetrics(scope)
+	if err != nil {
+		return nil, err
+	}
+
 	ingester := &ingester{
 		downsamplerAndWriter: downsamplerAndWriter,
 		opts:                 opts,
 		logger:               opts.InstrumentOptions.Logger(),
 		tagOpts:              tagOpts,
-		metrics: newCarbonIngesterMetrics(
-			opts.InstrumentOptions.MetricsScope()),
-		lineResourcesPool: resourcePool,
+		metrics:              metrics,
+		lineResourcesPool:    resourcePool,
 	}
+
 	// No need to retain watch as NamespaceWatcher.Close() will handle closing any watches
 	// generated by creating listeners.
 	clusterNamespacesWatcher.RegisterListener(ingester)
@@ -170,7 +176,7 @@ func (i *ingester) OnUpdate(clusterNamespaces m3.ClusterNamespaces) {
 		return
 	}
 
-	compiledRules, err := i.compileRulesWithLock(*rules)
+	compiledRules, err := compileRules(*rules)
 	if err != nil {
 		i.logger.Error("failed to compile latest rules. continuing to use existing carbon ingestion "+
 			"rules", zap.Error(err))
@@ -401,10 +407,8 @@ func (i *ingester) writeWithOptions(
 		return err
 	}
 
-	err = i.downsamplerAndWriter.Write(
-		ctx, tags, resources.datapoints, xtime.Second, nil, opts,
-	)
-
+	err = i.downsamplerAndWriter.Write(ctx, tags, resources.datapoints,
+		xtime.Second, nil, opts)
 	if err != nil {
 		i.logger.Error("err writing carbon metric",
 			zap.String("name", string(resources.name)), zap.Error(err))
@@ -419,18 +423,26 @@ func (i *ingester) Close() {
 	// We don't maintain any state in-between connections so there is nothing to do here.
 }
 
-func newCarbonIngesterMetrics(m tally.Scope) carbonIngesterMetrics {
-	return carbonIngesterMetrics{
-		success:   m.Counter("success"),
-		err:       m.Counter("error"),
-		malformed: m.Counter("malformed"),
-	}
+type carbonIngesterMetrics struct {
+	success       tally.Counter
+	err           tally.Counter
+	malformed     tally.Counter
+	ingestLatency tally.Histogram
+	writeLatency  tally.Histogram
 }
 
-type carbonIngesterMetrics struct {
-	success   tally.Counter
-	err       tally.Counter
-	malformed tally.Counter
+func newCarbonIngesterMetrics(scope tally.Scope) (carbonIngesterMetrics, error) {
+	buckets, err := ingest.NewLatencyBuckets()
+	if err != nil {
+		return carbonIngesterMetrics{}, err
+	}
+	return carbonIngesterMetrics{
+		success:       scope.Counter("success"),
+		err:           scope.Counter("error"),
+		malformed:     scope.Counter("malformed"),
+		writeLatency:  scope.SubScope("write").Histogram("latency", buckets.WriteLatencyBuckets),
+		ingestLatency: scope.SubScope("ingest").Histogram("latency", buckets.IngestLatencyBuckets),
+	}, nil
 }
 
 // GenerateTagsFromName accepts a carbon metric name and blows it up into a list of
@@ -519,7 +531,7 @@ func generateTagsFromName(
 // Note that only one rule will be applied per metric and rules are applied
 // such that the first one that matches takes precedence. As a result we need
 // to make sure to maintain the order of the rules when we generate the compiled ones.
-func (i *ingester) compileRulesWithLock(rules CarbonIngesterRules) ([]ruleAndRegex, error) {
+func compileRules(rules CarbonIngesterRules) ([]ruleAndRegex, error) {
 	var compiledRules []ruleAndRegex
 	for _, rule := range rules.Rules {
 		compiled, err := regexp.Compile(rule.Pattern)

src/cmd/services/m3coordinator/ingest/carbon/ingest_test.go

@@ -341,7 +341,7 @@ func TestIngesterNoStaticRules(t *testing.T) {
 	var expectationErr error
 	mockDownsamplerAndWriter, found := newMockDownsamplerAndWriter(ctrl, func(mappingRules []downsample.AutoMappingRule) {
 		if len(mappingRules) != 1 {
-			expectationErr = errors.New(fmt.Sprintf("expected: len(DownsampleMappingRules) == 1, got: %v", len(mappingRules)))
+			expectationErr = fmt.Errorf("expected: len(DownsampleMappingRules) == 1, got: %v", len(mappingRules))
 		}
 		policies := mappingRules[0].Policies
 
@@ -350,7 +350,7 @@ func TestIngesterNoStaticRules(t *testing.T) {
 		}
 		expectedPolicy := policy.NewStoragePolicy(10*time.Second, xtime.Second, 48*time.Hour)
 		if ok := expectedPolicy == policies[0]; !ok {
-			expectationErr = errors.New(fmt.Sprintf("expected storage policy: %+v, got: %+v", expectedPolicy, policies[0]))
+			expectationErr = fmt.Errorf("expected storage policy: %+v, got: %+v", expectedPolicy, policies[0])
 		}
 	})
 
@@ -416,11 +416,11 @@ func TestIngesterNoStaticRules(t *testing.T) {
 		}
 		expectedPolicy := policy.NewStoragePolicy(10*time.Second, xtime.Second, 48*time.Hour)
 		if ok := expectedPolicy == policies[0]; !ok {
-			expectationErr = errors.New(fmt.Sprintf("expected storage policy: %+v, got: %+v", expectedPolicy, policies[0]))
+			expectationErr = fmt.Errorf("expected storage policy: %+v, got: %+v", expectedPolicy, policies[0])
 		}
 		expectedPolicy = policy.NewStoragePolicy(1*time.Minute, xtime.Second, 168*time.Hour)
 		if ok := expectedPolicy == policies[1]; !ok {
-			expectationErr = errors.New(fmt.Sprintf("expected storage policy: %+v, got: %+v", expectedPolicy, policies[1]))
+			expectationErr = fmt.Errorf("expected storage policy: %+v, got: %+v", expectedPolicy, policies[1])
 		}
 	})
 

src/cmd/services/m3coordinator/ingest/metrics.go

@@ -0,0 +1,90 @@
+// Copyright (c) 2020 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package ingest
+
+import (
+	"time"
+
+	"github.com/uber-go/tally"
+)
+
+// LatencyBuckets are a set of latency buckets useful for measuring latencies.
+type LatencyBuckets struct {
+	WriteLatencyBuckets  tally.DurationBuckets
+	IngestLatencyBuckets tally.DurationBuckets
+}
+
+// NewLatencyBuckets returns write and ingest latency buckets useful for
+// measuring ingest latency (i.e. time from datapoint/sample created to time
+// ingested) and write latency (i.e. time from received a sample from remote
+// source to completion of that write locally).
+func NewLatencyBuckets() (LatencyBuckets, error) {
+	upTo1sBuckets, err := tally.LinearDurationBuckets(0, 100*time.Millisecond, 10)
+	if err != nil {
+		return LatencyBuckets{}, err
+	}
+
+	upTo10sBuckets, err := tally.LinearDurationBuckets(time.Second, 500*time.Millisecond, 18)
+	if err != nil {
+		return LatencyBuckets{}, err
+	}
+
+	upTo60sBuckets, err := tally.LinearDurationBuckets(10*time.Second, 5*time.Second, 11)
+	if err != nil {
+		return LatencyBuckets{}, err
+	}
+
+	upTo60mBuckets, err := tally.LinearDurationBuckets(0, 5*time.Minute, 12)
+	if err != nil {
+		return LatencyBuckets{}, err
+	}
+	upTo60mBuckets = upTo60mBuckets[1:] // Remove the first 0s to get 5 min aligned buckets
+
+	upTo6hBuckets, err := tally.LinearDurationBuckets(time.Hour, 30*time.Minute, 12)
+	if err != nil {
+		return LatencyBuckets{}, err
+	}
+
+	upTo24hBuckets, err := tally.LinearDurationBuckets(6*time.Hour, time.Hour, 19)
+	if err != nil {
+		return LatencyBuckets{}, err
+	}
+	upTo24hBuckets = upTo24hBuckets[1:] // Remove the first 6h to get 1 hour aligned buckets
+
+	var writeLatencyBuckets tally.DurationBuckets
+	writeLatencyBuckets = append(writeLatencyBuckets, upTo1sBuckets...)
+	writeLatencyBuckets = append(writeLatencyBuckets, upTo10sBuckets...)
+	writeLatencyBuckets = append(writeLatencyBuckets, upTo60sBuckets...)
+	writeLatencyBuckets = append(writeLatencyBuckets, upTo60mBuckets...)
+
+	var ingestLatencyBuckets tally.DurationBuckets
+	ingestLatencyBuckets = append(ingestLatencyBuckets, upTo1sBuckets...)
+	ingestLatencyBuckets = append(ingestLatencyBuckets, upTo10sBuckets...)
+	ingestLatencyBuckets = append(ingestLatencyBuckets, upTo60sBuckets...)
+	ingestLatencyBuckets = append(ingestLatencyBuckets, upTo60mBuckets...)
+	ingestLatencyBuckets = append(ingestLatencyBuckets, upTo6hBuckets...)
+	ingestLatencyBuckets = append(ingestLatencyBuckets, upTo24hBuckets...)
+
+	return LatencyBuckets{
+		WriteLatencyBuckets:  writeLatencyBuckets,
+		IngestLatencyBuckets: ingestLatencyBuckets,
+	}, nil
+}

src/cmd/services/m3coordinator/ingest/metrics_test.go

@@ -0,0 +1,49 @@
+// Copyright (c) 2020 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package ingest
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestLatencyBuckets(t *testing.T) {
+	buckets, err := NewLatencyBuckets()
+	require.NoError(t, err)
+
+	// NB(r): Bucket length is tested just to sanity check how many buckets we are creating
+	require.Equal(t, 50, len(buckets.WriteLatencyBuckets.AsDurations()))
+
+	// NB(r): Bucket values are tested to sanity check they look right
+	expected := "[0s 100ms 200ms 300ms 400ms 500ms 600ms 700ms 800ms 900ms 1s 1.5s 2s 2.5s 3s 3.5s 4s 4.5s 5s 5.5s 6s 6.5s 7s 7.5s 8s 8.5s 9s 9.5s 10s 15s 20s 25s 30s 35s 40s 45s 50s 55s 1m0s 5m0s 10m0s 15m0s 20m0s 25m0s 30m0s 35m0s 40m0s 45m0s 50m0s 55m0s]"
+	actual := fmt.Sprintf("%v", buckets.WriteLatencyBuckets.AsDurations())
+	require.Equal(t, expected, actual)
+
+	// NB(r): Bucket length is tested just to sanity check how many buckets we are creating
+	require.Equal(t, 80, len(buckets.IngestLatencyBuckets.AsDurations()))
+
+	// NB(r): Bucket values are tested to sanity check they look right
+	expected = "[0s 100ms 200ms 300ms 400ms 500ms 600ms 700ms 800ms 900ms 1s 1.5s 2s 2.5s 3s 3.5s 4s 4.5s 5s 5.5s 6s 6.5s 7s 7.5s 8s 8.5s 9s 9.5s 10s 15s 20s 25s 30s 35s 40s 45s 50s 55s 1m0s 5m0s 10m0s 15m0s 20m0s 25m0s 30m0s 35m0s 40m0s 45m0s 50m0s 55m0s 1h0m0s 1h30m0s 2h0m0s 2h30m0s 3h0m0s 3h30m0s 4h0m0s 4h30m0s 5h0m0s 5h30m0s 6h0m0s 6h30m0s 7h0m0s 8h0m0s 9h0m0s 10h0m0s 11h0m0s 12h0m0s 13h0m0s 14h0m0s 15h0m0s 16h0m0s 17h0m0s 18h0m0s 19h0m0s 20h0m0s 21h0m0s 22h0m0s 23h0m0s 24h0m0s]"
+	actual = fmt.Sprintf("%v", buckets.IngestLatencyBuckets.AsDurations())
+	require.Equal(t, expected, actual)
+}

src/dbnode/integration/index_single_node_high_concurrency_test.go

@@ -26,6 +26,7 @@ import (
 	"fmt"
 	"math/rand"
 	"strconv"
+	"strings"
 	"sync"
 	"testing"
 	"time"
@@ -377,7 +378,7 @@ func testIndexSingleNodeHighConcurrency(
 		// Now check all of them are individually indexed.
 		var (
 			fetchWg        sync.WaitGroup
-			notIndexedErrs []error
+			notIndexedErrs []string
 			notIndexedLock sync.Mutex
 		)
 		for i := 0; i < opts.concurrencyEnqueueWorker; i++ {
@@ -401,10 +402,18 @@ func testIndexSingleNodeHighConcurrency(
 							found := isIndexed(t, session, md.ID(), id, tags)
 							return found
 						}, 30*time.Second)
+
+						// Final check to get the corrersponding error/mismatch.
+						indexed, err := isIndexedChecked(t, session, md.ID(), id, tags)
 						if !indexed {
-							err := fmt.Errorf("not indexed series: i=%d, j=%d", i, j)
+							if err != nil {
+								err = fmt.Errorf("not indexed: i=%d, j=%d, err=%v", i, j, err)
+							} else {
+								err = fmt.Errorf("not indexed: i=%d, j=%d, err=none", i, j)
+							}
+
 							notIndexedLock.Lock()
-							notIndexedErrs = append(notIndexedErrs, err)
+							notIndexedErrs = append(notIndexedErrs, err.Error())
 							notIndexedLock.Unlock()
 						}
 					})
@@ -414,7 +423,8 @@ func testIndexSingleNodeHighConcurrency(
 		fetchWg.Wait()
 
 		require.Equal(t, 0, len(notIndexedErrs),
-			fmt.Sprintf("not indexed errors: %v", notIndexedErrs[:min(5, len(notIndexedErrs))]))
+			fmt.Sprintf("not indexed errors: [%v]",
+				strings.Join(notIndexedErrs[:min(5, len(notIndexedErrs))], ", ")))
 	}
 
 	log.Info("data indexing verify done", zap.Duration("took", time.Since(start)))

src/dbnode/server/server.go

@@ -1585,7 +1585,6 @@ func withEncodingAndPoolingOptions(
 				SetInstrumentOptions(iopts)).
 		SetFSTSegmentOptions(
 			opts.IndexOptions().FSTSegmentOptions().
-				SetPostingsListPool(postingsList).
 				SetInstrumentOptions(iopts).
 				SetContextPool(opts.ContextPool())).
 		SetSegmentBuilderOptions(