[dbnode] Add index snapshotting

scripts/docker-integration-tests/simple/test.sh

@@ -142,25 +142,25 @@ curl -vvvsS -X POST 0.0.0.0:9003/writetagged -d '{
   }
 }'
 
+function test_query_result {
+  queryResult=$(curl -sSf -X POST 0.0.0.0:9003/query -d '{
+    "namespace": "unagg",
+    "query": {
+      "regexp": {
+        "field": "city",
+        "regexp": ".*"
+      }
+    },
+    "rangeStart": 0,
+    "rangeEnd":'"$(date +"%s")"'
+  }' | jq '.results | length')
+  test "$queryResult" -eq "1"
+  return $?
+}
+
 echo "Read data"
-queryResult=$(curl -sSf -X POST 0.0.0.0:9003/query -d '{
-  "namespace": "unagg",
-  "query": {
-    "regexp": {
-      "field": "city",
-      "regexp": ".*"
-    }
-  },
-  "rangeStart": 0,
-  "rangeEnd":'"$(date +"%s")"'
-}' | jq '.results | length')
-
-if [ "$queryResult" -lt 1 ]; then
-  echo "Result not found"
-  exit 1
-else
-  echo "Result found"
-fi
+ATTEMPTS=10 MAX_TIMEOUT=1 TIMEOUT=.1 retry_with_backoff \
+  test_query_result
 
 echo "Deleting placement"
 curl -vvvsSf -X DELETE 0.0.0.0:7201/api/v1/services/m3db/placement

src/cmd/tools/read_index_segments/main/main.go

@@ -154,7 +154,7 @@ func readNamespaceSegments(
 ) {
 	var (
 		infoFiles = fs.ReadIndexInfoFiles(fsOpts.FilePathPrefix(), nsID,
-			fsOpts.InfoReaderBufferSize())
+			fsOpts.InfoReaderBufferSize(), persist.FileSetFlushType)
 		wg sync.WaitGroup
 	)
 

src/dbnode/digest/buffer.go

@@ -22,6 +22,7 @@ package digest
 
 import (
 	"encoding/binary"
+	"errors"
 	"os"
 )
 
@@ -33,6 +34,9 @@ const (
 var (
 	// Endianness is little endian
 	endianness = binary.LittleEndian
+	// errBufferSizeTooSmall is for when the buffer passed in to be
+	// converted to a digest buffer is of insufficient size.
+	errBufferSizeTooSmall = errors.New("buffer size too small")
 )
 
 // Buffer is a byte slice that facilitates digest reading and writing.
@@ -70,6 +74,9 @@ func (b Buffer) ReadDigestFromFile(fd *os.File) (uint32, error) {
 }
 
 // ToBuffer converts a byte slice to a digest buffer.
-func ToBuffer(buf []byte) Buffer {
-	return Buffer(buf[:DigestLenBytes])
+func ToBuffer(buf []byte) (Buffer, error) {
+	if len(buf) < DigestLenBytes {
+		return nil, errBufferSizeTooSmall
+	}
+	return Buffer(buf[:DigestLenBytes]), nil
 }

src/dbnode/digest/buffer_test.go

@@ -52,10 +52,16 @@ func TestWriteDigestToFile(t *testing.T) {
 }
 
 func TestReadDigest(t *testing.T) {
-	buf := ToBuffer([]byte{0x0, 0x1, 0x0, 0x1, 0x0, 0x1})
+	buf, err := ToBuffer([]byte{0x0, 0x1, 0x0, 0x1, 0x0, 0x1})
+	require.NoError(t, err)
 	require.Equal(t, uint32(0x1000100), buf.ReadDigest())
 }
 
+func TestReadDigestInsufficientBytes(t *testing.T) {
+	_, err := ToBuffer([]byte{0x0})
+	require.Equal(t, err, errBufferSizeTooSmall)
+}
+
 func TestReadDigestFromFile(t *testing.T) {
 	fd := createTempFile(t)
 	defer func() {

src/dbnode/generated/proto/index/index.proto

@@ -13,6 +13,12 @@ message IndexVolumeInfo {
   int64 snapshotTime = 6;
   repeated SegmentInfo segments = 7;
   google.protobuf.StringValue indexVolumeType = 8;
+  bytes snapshotID = 9;
+}
+
+enum SegmentState {
+    COMPACTABLE_SEGMENT_STATE = 0;
+    FROZEN_SEGMENT_STATE = 1;
 }
 
 message SegmentInfo {
@@ -21,6 +27,7 @@ message SegmentInfo {
   int64 minorVersion = 3;
   bytes metadata = 4;
   repeated SegmentFileInfo files = 5;
+  SegmentState segmentState = 6;
 }
 
 message SegmentFileInfo {