diff --git a/examples_test.go b/examples_test.go index 66d12bd0e..6202464d4 100644 --- a/examples_test.go +++ b/examples_test.go @@ -321,7 +321,7 @@ func ExampleNewHighlight() { panic(err) } - fmt.Println(searchResults.Hits[0].Fragments["Name"][0]) + fmt.Println(searchResults.Hits[0].Fragments["`Name`"][0]) // Output: // great nameless one } @@ -335,7 +335,7 @@ func ExampleNewHighlightWithStyle() { panic(err) } - fmt.Println(searchResults.Hits[0].Fragments["Name"][0]) + fmt.Println(searchResults.Hits[0].Fragments["`Name`"][0]) // Output: // great nameless one } @@ -446,7 +446,7 @@ func ExampleSearchRequest_SortByCustom() { searchRequest := NewSearchRequest(query) searchRequest.SortByCustom(search.SortOrder{ &search.SortField{ - Field: "Age", + Field: "`Age`", Missing: search.SortFieldMissingFirst, }, &search.SortDocID{}, diff --git a/http/handlers_test.go b/http/handlers_test.go index aff659ccb..f21e994e8 100644 --- a/http/handlers_test.go +++ b/http/handlers_test.go @@ -292,9 +292,9 @@ func TestHandlers(t *testing.T) { }, Status: http.StatusOK, ResponseMatch: map[string]bool{ - `"id":"a"`: true, - `"body":"test"`: true, - `"name":"a"`: true, + "\"id\":\"a\"": true, + "\"`body`\":\"test\"": true, + "\"`name`\":\"a\"": true, }, }, { @@ -483,10 +483,10 @@ func TestHandlers(t *testing.T) { }, Status: http.StatusOK, ResponseMatch: map[string]bool{ - `"fields":`: true, - `"name"`: true, - `"body"`: true, - `"_all"`: true, + "\"fields\"": true, + "\"`name`\"": true, + "\"`body`\"": true, + "\"_all\"": true, }, }, { diff --git a/index_impl.go b/index_impl.go index c5a0c46f4..8e30b8ebf 100644 --- a/index_impl.go +++ b/index_impl.go @@ -34,6 +34,7 @@ import ( "github.com/blevesearch/bleve/v2/search/collector" "github.com/blevesearch/bleve/v2/search/facet" "github.com/blevesearch/bleve/v2/search/highlight" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -631,7 +632,7 @@ func LoadAndHighlightFields(hit *search.DocumentMatch, req *SearchRequest, fieldsToLoad := deDuplicate(req.Fields) for _, f := range fieldsToLoad { doc.VisitFields(func(docF index.Field) { - if f == "*" || docF.Name() == f { + if f == "*" || docF.Name() == util.CleansePath(f) { var value interface{} switch docF := docF.(type) { case index.TextField: @@ -683,7 +684,7 @@ func LoadAndHighlightFields(hit *search.DocumentMatch, req *SearchRequest, } } for _, hf := range highlightFields { - highlighter.BestFragmentsInField(hit, doc, hf, 1) + highlighter.BestFragmentsInField(hit, doc, util.CleansePath(hf), 1) } } } else if doc == nil { @@ -737,6 +738,7 @@ func (i *indexImpl) FieldDict(field string) (index.FieldDict, error) { return nil, err } + field = util.CleansePath(field) fieldDict, err := indexReader.FieldDict(field) if err != nil { i.mutex.RUnlock() @@ -764,6 +766,7 @@ func (i *indexImpl) FieldDictRange(field string, startTerm []byte, endTerm []byt return nil, err } + field = util.CleansePath(field) fieldDict, err := indexReader.FieldDictRange(field, startTerm, endTerm) if err != nil { i.mutex.RUnlock() @@ -791,6 +794,7 @@ func (i *indexImpl) FieldDictPrefix(field string, termPrefix []byte) (index.Fiel return nil, err } + field = util.CleansePath(field) fieldDict, err := indexReader.FieldDictPrefix(field, termPrefix) if err != nil { i.mutex.RUnlock() diff --git a/index_test.go b/index_test.go index 0c89b4e6d..c63a45e28 100644 --- a/index_test.go +++ b/index_test.go @@ -24,7 +24,6 @@ import ( "os" "path/filepath" "reflect" - "sort" "strconv" "strings" "sync" @@ -199,7 +198,7 @@ func TestCrud(t *testing.T) { } foundNameField := false doc.VisitFields(func(field index.Field) { - if field.Name() == "name" && string(field.Value()) == "marty" { + if field.Name() == "`name`" && string(field.Value()) == "marty" { foundNameField = true } }) @@ -212,9 +211,9 @@ func TestCrud(t *testing.T) { t.Fatal(err) } expectedFields := map[string]bool{ - "_all": false, - "name": false, - "desc": false, + "_all": false, + "`name`": false, + "`desc`": false, } if len(fields) < len(expectedFields) { t.Fatalf("expected %d fields got %d", len(expectedFields), len(fields)) @@ -399,10 +398,11 @@ func TestBytesRead(t *testing.T) { if err != nil { t.Error(err) } + stats, _ := idx.StatsMap()["index"].(map[string]interface{}) prevBytesRead, _ := stats["num_bytes_read_at_query_time"].(uint64) - if prevBytesRead != 32349 && res.BytesRead == prevBytesRead { - t.Fatalf("expected bytes read for query string 32349, got %v", + if prevBytesRead != 32475 && res.BytesRead == prevBytesRead { + t.Fatalf("expected bytes read for query string 32475, got %v", prevBytesRead) } @@ -580,8 +580,8 @@ func TestBytesReadStored(t *testing.T) { stats, _ := idx.StatsMap()["index"].(map[string]interface{}) bytesRead, _ := stats["num_bytes_read_at_query_time"].(uint64) - if bytesRead != 25928 && bytesRead == res.BytesRead { - t.Fatalf("expected the bytes read stat to be around 25928, got %v", bytesRead) + if bytesRead != 26054 && bytesRead == res.BytesRead { + t.Fatalf("expected the bytes read stat to be around 26054, got %v", bytesRead) } prevBytesRead := bytesRead @@ -651,8 +651,8 @@ func TestBytesReadStored(t *testing.T) { stats, _ = idx1.StatsMap()["index"].(map[string]interface{}) bytesRead, _ = stats["num_bytes_read_at_query_time"].(uint64) - if bytesRead != 18114 && bytesRead == res.BytesRead { - t.Fatalf("expected the bytes read stat to be around 18114, got %v", bytesRead) + if bytesRead != 18240 && bytesRead == res.BytesRead { + t.Fatalf("expected the bytes read stat to be around 18240, got %v", bytesRead) } prevBytesRead = bytesRead @@ -920,17 +920,17 @@ func TestStoredFieldPreserved(t *testing.T) { if len(res.Hits) != 1 { t.Fatalf("expected 1 hit, got %d", len(res.Hits)) } - if res.Hits[0].Fields["name"] != "Marty" { - t.Errorf("expected 'Marty' got '%s'", res.Hits[0].Fields["name"]) + if res.Hits[0].Fields["`name`"] != "Marty" { + t.Errorf("expected 'Marty' got '%s'", res.Hits[0].Fields["`name`"]) } - if res.Hits[0].Fields["desc"] != "GopherCON India" { - t.Errorf("expected 'GopherCON India' got '%s'", res.Hits[0].Fields["desc"]) + if res.Hits[0].Fields["`desc`"] != "GopherCON India" { + t.Errorf("expected 'GopherCON India' got '%s'", res.Hits[0].Fields["`desc`"]) } - if res.Hits[0].Fields["num"] != float64(1) { - t.Errorf("expected '1' got '%v'", res.Hits[0].Fields["num"]) + if res.Hits[0].Fields["`num`"] != float64(1) { + t.Errorf("expected '1' got '%v'", res.Hits[0].Fields["`num`"]) } - if res.Hits[0].Fields["bool"] != true { - t.Errorf("expected 'true' got '%v'", res.Hits[0].Fields["bool"]) + if res.Hits[0].Fields["`bool`"] != true { + t.Errorf("expected 'true' got '%v'", res.Hits[0].Fields["`bool`"]) } } @@ -1185,7 +1185,7 @@ func TestSortMatchSearch(t *testing.T) { } prev := "" for _, hit := range sr.Hits { - val := hit.Fields["Day"].(string) + val := hit.Fields["`Day`"].(string) if prev > val { t.Errorf("Hits must be sorted by 'Day'. Found '%s' before '%s'", prev, val) } @@ -1533,14 +1533,14 @@ func TestTermVectorArrayPositions(t *testing.T) { if results.Total != 1 { t.Fatalf("expected 1 result, got %d", results.Total) } - if len(results.Hits[0].Locations["Messages"]["second"]) < 1 { + if len(results.Hits[0].Locations["`Messages`"]["second"]) < 1 { t.Fatalf("expected at least one location") } - if len(results.Hits[0].Locations["Messages"]["second"][0].ArrayPositions) < 1 { + if len(results.Hits[0].Locations["`Messages`"]["second"][0].ArrayPositions) < 1 { t.Fatalf("expected at least one location array position") } - if results.Hits[0].Locations["Messages"]["second"][0].ArrayPositions[0] != 1 { - t.Fatalf("expected array position 1, got %d", results.Hits[0].Locations["Messages"]["second"][0].ArrayPositions[0]) + if results.Hits[0].Locations["`Messages`"]["second"][0].ArrayPositions[0] != 1 { + t.Fatalf("expected array position 1, got %d", results.Hits[0].Locations["`Messages`"]["second"][0].ArrayPositions[0]) } // repeat search for this document in Messages field @@ -1555,14 +1555,14 @@ func TestTermVectorArrayPositions(t *testing.T) { if results.Total != 1 { t.Fatalf("expected 1 result, got %d", results.Total) } - if len(results.Hits[0].Locations["Messages"]["third"]) < 1 { + if len(results.Hits[0].Locations["`Messages`"]["third"]) < 1 { t.Fatalf("expected at least one location") } - if len(results.Hits[0].Locations["Messages"]["third"][0].ArrayPositions) < 1 { + if len(results.Hits[0].Locations["`Messages`"]["third"][0].ArrayPositions) < 1 { t.Fatalf("expected at least one location array position") } - if results.Hits[0].Locations["Messages"]["third"][0].ArrayPositions[0] != 2 { - t.Fatalf("expected array position 2, got %d", results.Hits[0].Locations["Messages"]["third"][0].ArrayPositions[0]) + if results.Hits[0].Locations["`Messages`"]["third"][0].ArrayPositions[0] != 2 { + t.Fatalf("expected array position 2, got %d", results.Hits[0].Locations["`Messages`"]["third"][0].ArrayPositions[0]) } err = index.Close() @@ -1611,14 +1611,21 @@ func TestDocumentStaticMapping(t *testing.T) { if err != nil { t.Fatal(err) } - sort.Strings(fields) - expectedFields := []string{"Date", "Numeric", "Text", "_all"} + expectedFields := map[string]bool{ + "`Date`": false, + "`Numeric`": false, + "`Text`": false, + "_all": false, + } if len(fields) < len(expectedFields) { - t.Fatalf("invalid field count: %d", len(fields)) + t.Fatalf("expected %d fields got %d", len(expectedFields), len(fields)) + } + for _, f := range fields { + expectedFields[f] = true } - for i, expected := range expectedFields { - if expected != fields[i] { - t.Fatalf("unexpected field[%d]: %s", i, fields[i]) + for ef, efp := range expectedFields { + if !efp { + t.Errorf("field %s is missing", ef) } } @@ -1791,13 +1798,13 @@ func TestDocumentFieldArrayPositionsBug295(t *testing.T) { if results.Total != 1 { t.Fatalf("expected 1 result, got %d", results.Total) } - if len(results.Hits[0].Locations["Messages"]["bleve"]) != 2 { - t.Fatalf("expected 2 locations of 'bleve', got %d", len(results.Hits[0].Locations["Messages"]["bleve"])) + if len(results.Hits[0].Locations["`Messages`"]["bleve"]) != 2 { + t.Fatalf("expected 2 locations of 'bleve', got %d", len(results.Hits[0].Locations["`Messages`"]["bleve"])) } - if results.Hits[0].Locations["Messages"]["bleve"][0].ArrayPositions[0] != 0 { + if results.Hits[0].Locations["`Messages`"]["bleve"][0].ArrayPositions[0] != 0 { t.Errorf("expected array position to be 0") } - if results.Hits[0].Locations["Messages"]["bleve"][1].ArrayPositions[0] != 1 { + if results.Hits[0].Locations["`Messages`"]["bleve"][1].ArrayPositions[0] != 1 { t.Errorf("expected array position to be 1") } @@ -1812,13 +1819,13 @@ func TestDocumentFieldArrayPositionsBug295(t *testing.T) { if results.Total != 1 { t.Fatalf("expected 1 result, got %d", results.Total) } - if len(results.Hits[0].Locations["Messages"]["bleve"]) != 2 { - t.Fatalf("expected 2 locations of 'bleve', got %d", len(results.Hits[0].Locations["Messages"]["bleve"])) + if len(results.Hits[0].Locations["`Messages`"]["bleve"]) != 2 { + t.Fatalf("expected 2 locations of 'bleve', got %d", len(results.Hits[0].Locations["`Messages`"]["bleve"])) } - if results.Hits[0].Locations["Messages"]["bleve"][0].ArrayPositions[0] != 0 { + if results.Hits[0].Locations["`Messages`"]["bleve"][0].ArrayPositions[0] != 0 { t.Errorf("expected array position to be 0") } - if results.Hits[0].Locations["Messages"]["bleve"][1].ArrayPositions[0] != 1 { + if results.Hits[0].Locations["`Messages`"]["bleve"][1].ArrayPositions[0] != 1 { t.Errorf("expected array position to be 1") } @@ -2389,7 +2396,7 @@ func TestBatchMerge(t *testing.T) { foundNameField := false doc.VisitFields(func(field index.Field) { - if field.Name() == "name" && string(field.Value()) == "blahblah" { + if field.Name() == "`name`" && string(field.Value()) == "blahblah" { foundNameField = true } }) @@ -2403,10 +2410,10 @@ func TestBatchMerge(t *testing.T) { } expectedFields := map[string]bool{ - "_all": false, - "name": false, - "desc": false, - "country": false, + "_all": false, + "`name`": false, + "`desc`": false, + "`country`": false, } if len(fields) < len(expectedFields) { t.Fatalf("expected %d fields got %d", len(expectedFields), len(fields)) @@ -2837,7 +2844,7 @@ func TestCopyIndex(t *testing.T) { } foundNameField := false doc.VisitFields(func(field index.Field) { - if field.Name() == "name" && string(field.Value()) == "tester" { + if field.Name() == "`name`" && string(field.Value()) == "tester" { foundNameField = true } }) @@ -2850,9 +2857,9 @@ func TestCopyIndex(t *testing.T) { t.Fatal(err) } expectedFields := map[string]bool{ - "_all": false, - "name": false, - "desc": false, + "_all": false, + "`name`": false, + "`desc`": false, } if len(fields) < len(expectedFields) { t.Fatalf("expected %d fields got %d", len(expectedFields), len(fields)) @@ -2906,7 +2913,7 @@ func TestCopyIndex(t *testing.T) { } copyFoundNameField := false copyDoc.VisitFields(func(field index.Field) { - if field.Name() == "name" && string(field.Value()) == "tester" { + if field.Name() == "`name`" && string(field.Value()) == "tester" { copyFoundNameField = true } }) @@ -2919,9 +2926,9 @@ func TestCopyIndex(t *testing.T) { t.Fatal(err) } copyExpectedFields := map[string]bool{ - "_all": false, - "name": false, - "desc": false, + "_all": false, + "`name`": false, + "`desc`": false, } if len(copyFields) < len(copyExpectedFields) { t.Fatalf("expected %d fields got %d", len(copyExpectedFields), len(copyFields)) diff --git a/mapping/document.go b/mapping/document.go index 3f3bbfd38..52aec3019 100644 --- a/mapping/document.go +++ b/mapping/document.go @@ -23,6 +23,7 @@ import ( "time" "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/util" ) // A DocumentMapping describes how a type of document @@ -97,14 +98,14 @@ func (dm *DocumentMapping) analyzerNameForPath(path string) string { } func (dm *DocumentMapping) fieldDescribedByPath(path string) *FieldMapping { - pathElements := decodePath(path) + pathElements := util.DecodePath(path) if len(pathElements) > 1 { // easy case, there is more than 1 path element remaining // the next path element must match a property name // at this level for propName, subDocMapping := range dm.Properties { if propName == pathElements[0] { - return subDocMapping.fieldDescribedByPath(encodePath(pathElements[1:])) + return subDocMapping.fieldDescribedByPath(util.EncodePath(pathElements[1:])) } } } @@ -115,10 +116,10 @@ func (dm *DocumentMapping) fieldDescribedByPath(path string) *FieldMapping { // first look for property name with empty field for propName, subDocMapping := range dm.Properties { - if propName == path { + if propName == pathElements[0] { // found property name match, now look at its fields for _, field := range subDocMapping.Fields { - if field.Name == "" || field.Name == path { + if field.Name == "" || field.Name == pathElements[0] { // match return field } @@ -127,10 +128,10 @@ func (dm *DocumentMapping) fieldDescribedByPath(path string) *FieldMapping { } // next, walk the properties again, looking for field overriding the name for propName, subDocMapping := range dm.Properties { - if propName != path { + if propName != pathElements[0] { // property name isn't a match, but field name could override it for _, field := range subDocMapping.Fields { - if field.Name == path { + if field.Name == pathElements[0] { return field } } @@ -145,7 +146,7 @@ func (dm *DocumentMapping) fieldDescribedByPath(path string) *FieldMapping { // closest document mapping to a field not explicitly mapped // use closestDocMapping func (dm *DocumentMapping) documentMappingForPath(path string) *DocumentMapping { - pathElements := decodePath(path) + pathElements := util.DecodePath(path) current := dm OUTER: for i, pathElement := range pathElements { @@ -173,7 +174,7 @@ OUTER: // closestDocMapping findest the most specific document mapping that matches // part of the provided path func (dm *DocumentMapping) closestDocMapping(path string) *DocumentMapping { - pathElements := decodePath(path) + pathElements := util.DecodePath(path) current := dm OUTER: for _, pathElement := range pathElements { @@ -361,7 +362,7 @@ func (dm *DocumentMapping) walkDocument(data interface{}, path []string, indexes // if the field has a name under the specified tag, prefer that tag := field.Tag.Get(structTagKey) - tagFieldName := parseTagName(tag) + tagFieldName := util.ParseTagName(tag) if tagFieldName == "-" { continue } @@ -406,7 +407,7 @@ func (dm *DocumentMapping) walkDocument(data interface{}, path []string, indexes } func (dm *DocumentMapping) processProperty(property interface{}, path []string, indexes []uint64, context *walkContext) { - pathString := encodePath(path) + pathString := util.EncodePath(path) // look to see if there is a mapping for this field subDocMapping := dm.documentMappingForPath(pathString) closestDocMapping := dm.closestDocMapping(pathString) diff --git a/mapping/field.go b/mapping/field.go index 511782acc..955928481 100644 --- a/mapping/field.go +++ b/mapping/field.go @@ -20,12 +20,12 @@ import ( "net" "time" - "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" - index "github.com/blevesearch/bleve_index_api" - "github.com/blevesearch/bleve/v2/analysis" + "github.com/blevesearch/bleve/v2/analysis/analyzer/keyword" "github.com/blevesearch/bleve/v2/document" "github.com/blevesearch/bleve/v2/geo" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" ) // control the default behavior for dynamic fields (those not explicitly mapped) @@ -377,11 +377,11 @@ func (fm *FieldMapping) analyzerForField(path []string, context *walkContext) an func getFieldName(pathString string, path []string, fieldMapping *FieldMapping) string { fieldName := pathString if fieldMapping.Name != "" { - parentName := "" + parentName := []string{} if len(path) > 1 { - parentName = encodePath(path[:len(path)-1]) + pathSeparator + parentName = path[:len(path)-1] } - fieldName = parentName + fieldMapping.Name + fieldName = util.EncodePath(append(parentName, fieldMapping.Name)) } return fieldName } diff --git a/mapping/index.go b/mapping/index.go index 1d982dd41..f0be5446d 100644 --- a/mapping/index.go +++ b/mapping/index.go @@ -17,13 +17,14 @@ package mapping import ( "encoding/json" "fmt" - index "github.com/blevesearch/bleve_index_api" "github.com/blevesearch/bleve/v2/analysis" "github.com/blevesearch/bleve/v2/analysis/analyzer/standard" "github.com/blevesearch/bleve/v2/analysis/datetime/optional" "github.com/blevesearch/bleve/v2/document" "github.com/blevesearch/bleve/v2/registry" + "github.com/blevesearch/bleve/v2/util" + index "github.com/blevesearch/bleve_index_api" ) var MappingJSONStrict = false @@ -310,7 +311,7 @@ func (im *IndexMappingImpl) determineType(data interface{}) string { } // now see if we can find a type using the mapping - typ, ok := mustString(lookupPropertyPath(data, im.TypeField)) + typ, ok := util.LookupPropertyPathStr(data, im.TypeField) if ok { return typ } @@ -375,7 +376,7 @@ func (im *IndexMappingImpl) AnalyzerNameForPath(path string) string { } // next we will try default analyzers for the path - pathDecoded := decodePath(path) + pathDecoded := util.DecodePath(path) for _, docMapping := range im.TypeMapping { rv := docMapping.defaultAnalyzerName(pathDecoded) if rv != "" { diff --git a/mapping/mapping_test.go b/mapping/mapping_test.go index e0151af7a..aeedfca71 100644 --- a/mapping/mapping_test.go +++ b/mapping/mapping_test.go @@ -105,10 +105,10 @@ func TestMappingStructWithJSONTags(t *testing.T) { foundNoJSONName := false count := 0 for _, f := range doc.Fields { - if f.Name() == "name" { + if f.Name() == "`name`" { foundJSONName = true } - if f.Name() == "NoJSONTag" { + if f.Name() == "`NoJSONTag`" { foundNoJSONName = true } count++ @@ -145,10 +145,10 @@ func TestMappingStructWithJSONTagsOneDisabled(t *testing.T) { foundNoJSONName := false count := 0 for _, f := range doc.Fields { - if f.Name() == "name" { + if f.Name() == "`name`" { foundJSONName = true } - if f.Name() == "NoJSONTag" { + if f.Name() == "`NoJSONTag`" { foundNoJSONName = true } count++ @@ -185,10 +185,10 @@ func TestMappingStructWithAlternateTags(t *testing.T) { foundNoBLEVEName := false count := 0 for _, f := range doc.Fields { - if f.Name() == "name" { + if f.Name() == "`name`" { foundBLEVEName = true } - if f.Name() == "NoBLEVETag" { + if f.Name() == "`NoBLEVETag`" { foundNoBLEVEName = true } count++ @@ -227,10 +227,10 @@ func TestMappingStructWithAlternateTagsTwoDisabled(t *testing.T) { foundNoBLEVEName := false count := 0 for _, f := range doc.Fields { - if f.Name() == "name" { + if f.Name() == "`name`" { foundBLEVEName = true } - if f.Name() == "NoBLEVETag" { + if f.Name() == "`NoBLEVETag`" { foundNoBLEVEName = true } count++ @@ -266,7 +266,7 @@ func TestMappingStructWithPointerToString(t *testing.T) { found := false count := 0 for _, f := range doc.Fields { - if f.Name() == "Name" { + if f.Name() == "`Name`" { found = true } count++ @@ -298,7 +298,7 @@ func TestMappingJSONWithNull(t *testing.T) { found := false count := 0 for _, f := range doc.Fields { - if f.Name() == "name" { + if f.Name() == "`name`" { found = true } count++ @@ -504,10 +504,10 @@ func TestMappingBool(t *testing.T) { foundPProp := false count := 0 for _, f := range doc.Fields { - if f.Name() == "prop" { + if f.Name() == "`prop`" { foundProp = true } - if f.Name() == "pprop" { + if f.Name() == "`pprop`" { foundPProp = true } count++ @@ -735,17 +735,17 @@ func TestAnonymousStructFields(t *testing.T) { if len(doc.Fields) != 4 { t.Fatalf("expected 4 fields, got %d", len(doc.Fields)) } - if doc.Fields[0].Name() != "Contact0" { - t.Errorf("expected field named 'Contact0', got '%s'", doc.Fields[0].Name()) + if doc.Fields[0].Name() != "`Contact0`" { + t.Errorf("expected field named '`Contact0`', got '%s'", doc.Fields[0].Name()) } - if doc.Fields[1].Name() != "Name" { - t.Errorf("expected field named 'Name', got '%s'", doc.Fields[1].Name()) + if doc.Fields[1].Name() != "`Name`" { + t.Errorf("expected field named '`Name`', got '%s'", doc.Fields[1].Name()) } - if doc.Fields[2].Name() != "Contact2.Name" { - t.Errorf("expected field named 'Contact2.Name', got '%s'", doc.Fields[2].Name()) + if doc.Fields[2].Name() != "`Contact2`.`Name`" { + t.Errorf("expected field named '`Contact2`.`Name`', got '%s'", doc.Fields[2].Name()) } - if doc.Fields[3].Name() != "Contact3" { - t.Errorf("expected field named 'Contact3', got '%s'", doc.Fields[3].Name()) + if doc.Fields[3].Name() != "`Contact3`" { + t.Errorf("expected field named '`Contact3`', got '%s'", doc.Fields[3].Name()) } type AnotherThing struct { @@ -775,17 +775,17 @@ func TestAnonymousStructFields(t *testing.T) { if len(doc2.Fields) != 4 { t.Fatalf("expected 4 fields, got %d", len(doc2.Fields)) } - if doc2.Fields[0].Name() != "Alternate0" { - t.Errorf("expected field named 'Alternate0', got '%s'", doc2.Fields[0].Name()) + if doc2.Fields[0].Name() != "`Alternate0`" { + t.Errorf("expected field named '`Alternate0`', got '%s'", doc2.Fields[0].Name()) } - if doc2.Fields[1].Name() != "Alternate1.Name" { - t.Errorf("expected field named 'Name', got '%s'", doc2.Fields[1].Name()) + if doc2.Fields[1].Name() != "`Alternate1`.`Name`" { + t.Errorf("expected field named '`Alternte1`.`Name`', got '%s'", doc2.Fields[1].Name()) } - if doc2.Fields[2].Name() != "Alternate2.Name" { - t.Errorf("expected field named 'Alternate2.Name', got '%s'", doc2.Fields[2].Name()) + if doc2.Fields[2].Name() != "`Alternate2`.`Name`" { + t.Errorf("expected field named '`Alternate2`.`Name`', got '%s'", doc2.Fields[2].Name()) } - if doc2.Fields[3].Name() != "Alternate3" { - t.Errorf("expected field named 'Alternate3', got '%s'", doc2.Fields[3].Name()) + if doc2.Fields[3].Name() != "`Alternate3`" { + t.Errorf("expected field named '`Alternate3`', got '%s'", doc2.Fields[3].Name()) } } @@ -811,8 +811,8 @@ func TestAnonymousStructFieldWithJSONStructTagEmptString(t *testing.T) { if len(doc.Fields) != 1 { t.Fatalf("expected 1 field, got %d", len(doc.Fields)) } - if doc.Fields[0].Name() != "key" { - t.Errorf("expected field named 'key', got '%s'", doc.Fields[0].Name()) + if doc.Fields[0].Name() != "`key`" { + t.Errorf("expected field named '`key`', got '%s'", doc.Fields[0].Name()) } } @@ -968,7 +968,7 @@ func TestMappingForGeo(t *testing.T) { var foundGeo bool for _, f := range doc.Fields { - if f.Name() == "location" { + if f.Name() == "`location`" { foundGeo = true geoF, ok := f.(index.GeoPointField) if !ok { @@ -1030,8 +1030,8 @@ func TestMappingForTextMarshaler(t *testing.T) { if len(doc.Fields) != 1 { t.Fatalf("expected 1 field, got: %d", len(doc.Fields)) } - if doc.Fields[0].Name() != "Marshalable.Extra" { - t.Errorf("expected field to be named 'Marshalable.Extra', got: '%s'", doc.Fields[0].Name()) + if doc.Fields[0].Name() != "`Marshalable`.`Extra`" { + t.Errorf("expected field to be named '`Marshalable`.`Extra`', got: '%s'", doc.Fields[0].Name()) } if string(doc.Fields[0].Value()) != tm.Marshalable.Extra { t.Errorf("expected field value to be '%s', got: '%s'", tm.Marshalable.Extra, string(doc.Fields[0].Value())) @@ -1051,8 +1051,8 @@ func TestMappingForTextMarshaler(t *testing.T) { t.Fatalf("expected 1 field, got: %d", len(doc.Fields)) } - if doc.Fields[0].Name() != "Marshalable" { - t.Errorf("expected field to be named 'Marshalable', got: '%s'", doc.Fields[0].Name()) + if doc.Fields[0].Name() != "`Marshalable`" { + t.Errorf("expected field to be named '`Marshalable`', got: '%s'", doc.Fields[0].Name()) } want, err := tm.Marshalable.MarshalText() if err != nil { @@ -1167,7 +1167,7 @@ func TestWrongAnalyzerSearchableAs(t *testing.T) { indexMapping := NewIndexMapping() indexMapping.AddDocumentMapping("brewery", docMapping) - analyzerName := indexMapping.AnalyzerNameForPath("geo.geo.accuracy") + analyzerName := indexMapping.AnalyzerNameForPath("`geo`.`geo.accuracy`") if analyzerName != "xyz" { t.Errorf("expected analyzer name `xyz`, got `%s`", analyzerName) } @@ -1228,7 +1228,7 @@ func TestMappingArrayOfStringGeoPoints(t *testing.T) { } for _, f := range doc.Fields { - if f.Name() == "points" { + if f.Name() == "`points`" { geoF, ok := f.(*document.GeoPointField) if !ok { t.Errorf("expected a geopoint field!") diff --git a/search/collector/topn.go b/search/collector/topn.go index 4d19cd455..edb290549 100644 --- a/search/collector/topn.go +++ b/search/collector/topn.go @@ -22,6 +22,7 @@ import ( "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/size" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -180,6 +181,7 @@ func (hc *TopNCollector) Collect(ctx context.Context, searcher search.Searcher, } hc.updateFieldVisitor = func(field string, term []byte) { + field = util.CleansePath(field) if hc.facetsBuilder != nil { hc.facetsBuilder.UpdateVisitor(field, term) } diff --git a/search/facet/facet_builder_datetime.go b/search/facet/facet_builder_datetime.go index ff5167f21..a9990c05a 100644 --- a/search/facet/facet_builder_datetime.go +++ b/search/facet/facet_builder_datetime.go @@ -22,6 +22,7 @@ import ( "github.com/blevesearch/bleve/v2/numeric" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/size" + "github.com/blevesearch/bleve/v2/util" ) var reflectStaticSizeDateTimeFacetBuilder int @@ -52,7 +53,7 @@ type DateTimeFacetBuilder struct { func NewDateTimeFacetBuilder(field string, size int) *DateTimeFacetBuilder { return &DateTimeFacetBuilder{ size: size, - field: field, + field: util.CleansePath(field), termsCount: make(map[string]int), ranges: make(map[string]*dateTimeRange, 0), } diff --git a/search/facet/facet_builder_numeric.go b/search/facet/facet_builder_numeric.go index f19634d7b..c35b149e2 100644 --- a/search/facet/facet_builder_numeric.go +++ b/search/facet/facet_builder_numeric.go @@ -21,6 +21,7 @@ import ( "github.com/blevesearch/bleve/v2/numeric" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/size" + "github.com/blevesearch/bleve/v2/util" ) var reflectStaticSizeNumericFacetBuilder int @@ -51,7 +52,7 @@ type NumericFacetBuilder struct { func NewNumericFacetBuilder(field string, size int) *NumericFacetBuilder { return &NumericFacetBuilder{ size: size, - field: field, + field: util.CleansePath(field), termsCount: make(map[string]int), ranges: make(map[string]*numericRange, 0), } diff --git a/search/facet/facet_builder_terms.go b/search/facet/facet_builder_terms.go index c5a1c8318..04a8ac1ed 100644 --- a/search/facet/facet_builder_terms.go +++ b/search/facet/facet_builder_terms.go @@ -20,6 +20,7 @@ import ( "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/size" + "github.com/blevesearch/bleve/v2/util" ) var reflectStaticSizeTermsFacetBuilder int @@ -41,7 +42,7 @@ type TermsFacetBuilder struct { func NewTermsFacetBuilder(field string, size int) *TermsFacetBuilder { return &TermsFacetBuilder{ size: size, - field: field, + field: util.CleansePath(field), termsCount: make(map[string]int), } } diff --git a/search/query/bool_field.go b/search/query/bool_field.go index 5aa7bb8af..92b315b84 100644 --- a/search/query/bool_field.go +++ b/search/query/bool_field.go @@ -20,6 +20,7 @@ import ( "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -57,7 +58,10 @@ func (q *BoolFieldQuery) Searcher(ctx context.Context, i index.IndexReader, m ma field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() + } else { + field = util.CleansePath(field) } + term := "F" if q.Bool { term = "T" diff --git a/search/query/date_range.go b/search/query/date_range.go index ef18f2fb8..a7f567538 100644 --- a/search/query/date_range.go +++ b/search/query/date_range.go @@ -27,6 +27,7 @@ import ( "github.com/blevesearch/bleve/v2/registry" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -96,7 +97,9 @@ type DateRangeQuery struct { // NewDateRangeQuery creates a new Query for ranges // of date values. // Date strings are parsed using the DateTimeParser configured in the -// top-level config.QueryDateTimeParser +// +// top-level config.QueryDateTimeParser +// // Either, but not both endpoints can be nil. func NewDateRangeQuery(start, end time.Time) *DateRangeQuery { return NewDateRangeInclusiveQuery(start, end, nil, nil) @@ -105,7 +108,9 @@ func NewDateRangeQuery(start, end time.Time) *DateRangeQuery { // NewDateRangeInclusiveQuery creates a new Query for ranges // of date values. // Date strings are parsed using the DateTimeParser configured in the -// top-level config.QueryDateTimeParser +// +// top-level config.QueryDateTimeParser +// // Either, but not both endpoints can be nil. // startInclusive and endInclusive control inclusion of the endpoints. func NewDateRangeInclusiveQuery(start, end time.Time, startInclusive, endInclusive *bool) *DateRangeQuery { @@ -143,6 +148,8 @@ func (q *DateRangeQuery) Searcher(ctx context.Context, i index.IndexReader, m ma field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() + } else { + field = util.CleansePath(field) } return searcher.NewNumericRangeSearcher(ctx, i, min, max, q.InclusiveStart, q.InclusiveEnd, field, q.BoostVal.Value(), options) diff --git a/search/query/fuzzy.go b/search/query/fuzzy.go index f24eb0c20..2491e0227 100644 --- a/search/query/fuzzy.go +++ b/search/query/fuzzy.go @@ -20,6 +20,7 @@ import ( "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -74,6 +75,9 @@ func (q *FuzzyQuery) Searcher(ctx context.Context, i index.IndexReader, m mappin field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() + } else { + field = util.CleansePath(field) } + return searcher.NewFuzzySearcher(ctx, i, q.Term, q.Prefix, q.Fuzziness, field, q.BoostVal.Value(), options) } diff --git a/search/query/geo_boundingbox.go b/search/query/geo_boundingbox.go index ac9125393..954809d66 100644 --- a/search/query/geo_boundingbox.go +++ b/search/query/geo_boundingbox.go @@ -23,6 +23,7 @@ import ( "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -61,6 +62,8 @@ func (q *GeoBoundingBoxQuery) Searcher(ctx context.Context, i index.IndexReader, field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() + } else { + field = util.CleansePath(field) } if q.BottomRight[0] < q.TopLeft[0] { diff --git a/search/query/geo_boundingpolygon.go b/search/query/geo_boundingpolygon.go index 467f39b28..cb98a425b 100644 --- a/search/query/geo_boundingpolygon.go +++ b/search/query/geo_boundingpolygon.go @@ -23,6 +23,7 @@ import ( "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -59,6 +60,8 @@ func (q *GeoBoundingPolygonQuery) Searcher(ctx context.Context, i index.IndexRea field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() + } else { + field = util.CleansePath(field) } return searcher.NewGeoBoundedPolygonSearcher(ctx, i, q.Points, field, q.BoostVal.Value(), options) diff --git a/search/query/geo_distance.go b/search/query/geo_distance.go index f05bf6723..4b4ccf2b3 100644 --- a/search/query/geo_distance.go +++ b/search/query/geo_distance.go @@ -23,6 +23,7 @@ import ( "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -62,6 +63,8 @@ func (q *GeoDistanceQuery) Searcher(ctx context.Context, i index.IndexReader, m field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() + } else { + field = util.CleansePath(field) } dist, err := geo.ParseDistance(q.Distance) diff --git a/search/query/geo_shape.go b/search/query/geo_shape.go index a63ec80f7..65e323155 100644 --- a/search/query/geo_shape.go +++ b/search/query/geo_shape.go @@ -22,6 +22,7 @@ import ( "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -105,6 +106,8 @@ func (q *GeoShapeQuery) Searcher(ctx context.Context, i index.IndexReader, field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() + } else { + field = util.CleansePath(field) } return searcher.NewGeoShapeSearcher(ctx, i, q.Geometry.Shape, q.Geometry.Relation, field, diff --git a/search/query/ip_range.go b/search/query/ip_range.go index ba46f0b25..3f93eeab5 100644 --- a/search/query/ip_range.go +++ b/search/query/ip_range.go @@ -22,6 +22,7 @@ import ( "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -58,7 +59,10 @@ func (q *IPRangeQuery) Searcher(ctx context.Context, i index.IndexReader, m mapp field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() + } else { + field = util.CleansePath(field) } + _, ipNet, err := net.ParseCIDR(q.CIDR) if err != nil { ip := net.ParseIP(q.CIDR) diff --git a/search/query/match.go b/search/query/match.go index 61c00a003..34b5351cb 100644 --- a/search/query/match.go +++ b/search/query/match.go @@ -21,6 +21,7 @@ import ( "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -120,6 +121,8 @@ func (q *MatchQuery) Searcher(ctx context.Context, i index.IndexReader, m mappin field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() + } else { + field = util.CleansePath(field) } analyzerName := "" diff --git a/search/query/match_phrase.go b/search/query/match_phrase.go index fa8ac720b..8989efaf8 100644 --- a/search/query/match_phrase.go +++ b/search/query/match_phrase.go @@ -21,6 +21,7 @@ import ( "github.com/blevesearch/bleve/v2/analysis" "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -66,6 +67,8 @@ func (q *MatchPhraseQuery) Searcher(ctx context.Context, i index.IndexReader, m field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() + } else { + field = util.CleansePath(field) } analyzerName := "" diff --git a/search/query/multi_phrase.go b/search/query/multi_phrase.go index 2887be16a..b44ea6e54 100644 --- a/search/query/multi_phrase.go +++ b/search/query/multi_phrase.go @@ -22,6 +22,7 @@ import ( "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -57,7 +58,14 @@ func (q *MultiPhraseQuery) Boost() float64 { } func (q *MultiPhraseQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { - return searcher.NewMultiPhraseSearcher(ctx, i, q.Terms, q.Field, options) + field := q.Field + if q.Field == "" { + field = m.DefaultSearchField() + } else { + field = util.CleansePath(field) + } + + return searcher.NewMultiPhraseSearcher(ctx, i, q.Terms, field, options) } func (q *MultiPhraseQuery) Validate() error { diff --git a/search/query/numeric_range.go b/search/query/numeric_range.go index ad2474167..c78f38977 100644 --- a/search/query/numeric_range.go +++ b/search/query/numeric_range.go @@ -21,6 +21,7 @@ import ( "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -76,7 +77,10 @@ func (q *NumericRangeQuery) Searcher(ctx context.Context, i index.IndexReader, m field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() + } else { + field = util.CleansePath(field) } + return searcher.NewNumericRangeSearcher(ctx, i, q.Min, q.Max, q.InclusiveMin, q.InclusiveMax, field, q.BoostVal.Value(), options) } diff --git a/search/query/phrase.go b/search/query/phrase.go index 207e66b17..ac44f6288 100644 --- a/search/query/phrase.go +++ b/search/query/phrase.go @@ -22,6 +22,7 @@ import ( "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -54,7 +55,14 @@ func (q *PhraseQuery) Boost() float64 { } func (q *PhraseQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping.IndexMapping, options search.SearcherOptions) (search.Searcher, error) { - return searcher.NewPhraseSearcher(ctx, i, q.Terms, q.Field, options) + field := q.Field + if q.Field == "" { + field = m.DefaultSearchField() + } else { + field = util.CleansePath(field) + } + + return searcher.NewPhraseSearcher(ctx, i, q.Terms, field, options) } func (q *PhraseQuery) Validate() error { diff --git a/search/query/prefix.go b/search/query/prefix.go index debbbc1e3..656522833 100644 --- a/search/query/prefix.go +++ b/search/query/prefix.go @@ -20,6 +20,7 @@ import ( "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -59,6 +60,9 @@ func (q *PrefixQuery) Searcher(ctx context.Context, i index.IndexReader, m mappi field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() + } else { + field = util.CleansePath(field) } + return searcher.NewTermPrefixSearcher(ctx, i, q.Prefix, field, q.BoostVal.Value(), options) } diff --git a/search/query/regexp.go b/search/query/regexp.go index 6b3da9554..d4ee30163 100644 --- a/search/query/regexp.go +++ b/search/query/regexp.go @@ -21,6 +21,7 @@ import ( "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -62,6 +63,8 @@ func (q *RegexpQuery) Searcher(ctx context.Context, i index.IndexReader, m mappi field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() + } else { + field = util.CleansePath(field) } // require that pattern NOT be anchored to start and end of term. diff --git a/search/query/term.go b/search/query/term.go index 5c6af3962..2a85c041c 100644 --- a/search/query/term.go +++ b/search/query/term.go @@ -20,6 +20,7 @@ import ( "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -58,6 +59,9 @@ func (q *TermQuery) Searcher(ctx context.Context, i index.IndexReader, m mapping field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() + } else { + field = util.CleansePath(field) } + return searcher.NewTermSearcher(ctx, i, q.Term, field, q.BoostVal.Value(), options) } diff --git a/search/query/term_range.go b/search/query/term_range.go index 4dc3a34b7..54b0ab38a 100644 --- a/search/query/term_range.go +++ b/search/query/term_range.go @@ -21,6 +21,7 @@ import ( "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -76,7 +77,10 @@ func (q *TermRangeQuery) Searcher(ctx context.Context, i index.IndexReader, m ma field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() + } else { + field = util.CleansePath(field) } + var minTerm []byte if q.Min != "" { minTerm = []byte(q.Min) @@ -85,6 +89,7 @@ func (q *TermRangeQuery) Searcher(ctx context.Context, i index.IndexReader, m ma if q.Max != "" { maxTerm = []byte(q.Max) } + return searcher.NewTermRangeSearcher(ctx, i, minTerm, maxTerm, q.InclusiveMin, q.InclusiveMax, field, q.BoostVal.Value(), options) } diff --git a/search/query/wildcard.go b/search/query/wildcard.go index f04f3f2ed..2d747c5e2 100644 --- a/search/query/wildcard.go +++ b/search/query/wildcard.go @@ -21,6 +21,7 @@ import ( "github.com/blevesearch/bleve/v2/mapping" "github.com/blevesearch/bleve/v2/search" "github.com/blevesearch/bleve/v2/search/searcher" + "github.com/blevesearch/bleve/v2/util" index "github.com/blevesearch/bleve_index_api" ) @@ -81,6 +82,8 @@ func (q *WildcardQuery) Searcher(ctx context.Context, i index.IndexReader, m map field := q.FieldVal if q.FieldVal == "" { field = m.DefaultSearchField() + } else { + field = util.CleansePath(field) } regexpString := wildcardRegexpReplacer.Replace(q.Wildcard) diff --git a/search/sort.go b/search/sort.go index 3a744af99..995ee6e00 100644 --- a/search/sort.go +++ b/search/sort.go @@ -25,6 +25,7 @@ import ( "github.com/blevesearch/bleve/v2/geo" "github.com/blevesearch/bleve/v2/numeric" + "github.com/blevesearch/bleve/v2/util" ) var HighTerm = strings.Repeat(string(utf8.MaxRune), 3) @@ -64,6 +65,7 @@ func ParseSearchSortObj(input map[string]interface{}) (SearchSort, error) { if !ok { return nil, fmt.Errorf("search sort mode geo_distance must specify field") } + field = util.CleansePath(field) lon, lat, foundLocation := geo.ExtractGeoPoint(input["location"]) if !foundLocation { return nil, fmt.Errorf("unable to parse geo_distance location") @@ -89,6 +91,7 @@ func ParseSearchSortObj(input map[string]interface{}) (SearchSort, error) { if !ok { return nil, fmt.Errorf("search sort mode field must specify field") } + field = util.CleansePath(field) rv := &SortField{ Field: field, Desc: descending, @@ -156,7 +159,7 @@ func ParseSearchSortString(input string) SearchSort { } } return &SortField{ - Field: input, + Field: util.CleansePath(input), Desc: descending, } } diff --git a/search_test.go b/search_test.go index e4a880a9a..61d48551b 100644 --- a/search_test.go +++ b/search_test.go @@ -480,7 +480,7 @@ func TestNestedBooleanSearchers(t *testing.T) { doc := document.NewDocument(strconv.Itoa(i)) doc.Fields = []document.Field{ - document.NewTextFieldCustom("hostname", []uint64{}, []byte(hostname), + document.NewTextFieldCustom("`hostname`", []uint64{}, []byte(hostname), index.IndexField, &analysis.DefaultAnalyzer{ Tokenizer: single.NewSingleTokenTokenizer(), @@ -492,7 +492,7 @@ func TestNestedBooleanSearchers(t *testing.T) { } for k, v := range metadata { doc.AddField(document.NewTextFieldWithIndexingOptions( - fmt.Sprintf("metadata.%s", k), []uint64{}, []byte(v), index.IndexField)) + fmt.Sprintf("`metadata`.`%s`", k), []uint64{}, []byte(v), index.IndexField)) } doc.CompositeFields = []*document.CompositeField{ document.NewCompositeFieldWithIndexingOptions( @@ -640,9 +640,9 @@ func TestNestedBooleanMustNotSearcherUpsidedown(t *testing.T) { for i := 0; i < len(docs); i++ { doc := document.NewDocument(docs[i].id) doc.Fields = []document.Field{ - document.NewTextField("id", []uint64{}, []byte(docs[i].id)), - document.NewBooleanField("hasRole", []uint64{}, docs[i].hasRole), - document.NewTextField("investigationId", []uint64{}, []byte(docs[i].investigationId)), + document.NewTextField("`id`", []uint64{}, []byte(docs[i].id)), + document.NewBooleanField("`hasRole`", []uint64{}, docs[i].hasRole), + document.NewTextField("`investigationId`", []uint64{}, []byte(docs[i].investigationId)), } doc.CompositeFields = []*document.CompositeField{ @@ -776,10 +776,10 @@ func TestMultipleNestedBooleanMustNotSearchersOnScorch(t *testing.T) { doc := document.NewDocument("1-child-0") doc.Fields = []document.Field{ - document.NewTextField("id", []uint64{}, []byte("1-child-0")), - document.NewBooleanField("hasRole", []uint64{}, false), - document.NewTextField("roles", []uint64{}, []byte("R1")), - document.NewNumericField("type", []uint64{}, 0), + document.NewTextField("`id`", []uint64{}, []byte("1-child-0")), + document.NewBooleanField("`hasRole`", []uint64{}, false), + document.NewTextField("`roles`", []uint64{}, []byte("R1")), + document.NewNumericField("`type`", []uint64{}, 0), } doc.CompositeFields = []*document.CompositeField{ document.NewCompositeFieldWithIndexingOptions( @@ -821,9 +821,9 @@ func TestMultipleNestedBooleanMustNotSearchersOnScorch(t *testing.T) { for i := 0; i < len(docs); i++ { doc := document.NewDocument(docs[i].id) doc.Fields = []document.Field{ - document.NewTextField("id", []uint64{}, []byte(docs[i].id)), - document.NewBooleanField("hasRole", []uint64{}, docs[i].hasRole), - document.NewNumericField("type", []uint64{}, float64(docs[i].typ)), + document.NewTextField("`id`", []uint64{}, []byte(docs[i].id)), + document.NewBooleanField("`hasRole`", []uint64{}, docs[i].hasRole), + document.NewNumericField("`type`", []uint64{}, float64(docs[i].typ)), } doc.CompositeFields = []*document.CompositeField{ @@ -846,9 +846,9 @@ func TestMultipleNestedBooleanMustNotSearchersOnScorch(t *testing.T) { // Update 1st doc doc = document.NewDocument("1-child-0") doc.Fields = []document.Field{ - document.NewTextField("id", []uint64{}, []byte("1-child-0")), - document.NewBooleanField("hasRole", []uint64{}, false), - document.NewNumericField("type", []uint64{}, 0), + document.NewTextField("`id`", []uint64{}, []byte("1-child-0")), + document.NewBooleanField("`hasRole`", []uint64{}, false), + document.NewNumericField("`type`", []uint64{}, 0), } doc.CompositeFields = []*document.CompositeField{ document.NewCompositeFieldWithIndexingOptions( @@ -1254,7 +1254,7 @@ func TestDuplicateLocationsIssue1168(t *testing.T) { if err != nil { t.Fatalf("bleve search err: %v", err) } - if len(sres.Hits[0].Locations["name1"]["marty"]) != 1 { + if len(sres.Hits[0].Locations["`name1`"]["marty"]) != 1 { t.Fatalf("duplicate marty") } } @@ -1866,7 +1866,7 @@ func TestHightlightingWithHTMLCharacterFilter(t *testing.T) { } if len(searchResults.Hits) != 1 || - len(searchResults.Hits[0].Locations["content"][searchStr]) != 1 { + len(searchResults.Hits[0].Locations["`content`"][searchStr]) != 1 { t.Fatalf("Expected 1 hit with 1 location") } @@ -1877,8 +1877,8 @@ func TestHightlightingWithHTMLCharacterFilter(t *testing.T) { } expectedFragment := "<div> Welcome to blevesearch. </div>" - gotLocation := searchResults.Hits[0].Locations["content"]["blevesearch"][0] - gotFragment := searchResults.Hits[0].Fragments["content"][0] + gotLocation := searchResults.Hits[0].Locations["`content`"]["blevesearch"][0] + gotFragment := searchResults.Hits[0].Fragments["`content`"][0] if !reflect.DeepEqual(expectedLocation, gotLocation) { t.Fatalf("Mismatch in locations, got: %v, expected: %v", @@ -2100,3 +2100,56 @@ func TestGeoShapePolygonContainsPoint(t *testing.T) { } } } + +func TestMB55699(t *testing.T) { + // Unit test to demonstrate capability to differentiate between + // a nested field name and one that has a "." within it. + tmpIndexPath := createTmpIndexPath(t) + defer cleanupTmpIndexPath(t, tmpIndexPath) + idx, err := New(tmpIndexPath, NewIndexMapping()) + if err != nil { + t.Fatal(err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + docBytes := []byte(` + { + "x": { + "y": "1" + }, + "x.y": "2" + } + `) + var doc map[string]interface{} + if err = json.Unmarshal(docBytes, &doc); err != nil { + t.Fatal(err) + } + + if err = idx.Index("doc", doc); err != nil { + t.Fatal(err) + } + + q1 := query.NewMatchQuery("1") + q1.SetField("x.y") + if res, err := idx.Search(NewSearchRequest(q1)); err != nil || len(res.Hits) != 1 { + t.Fatalf("Expected x.y to contain 1") + } + q1.SetField("`x`.`y`") + if res, err := idx.Search(NewSearchRequest(q1)); err != nil || len(res.Hits) != 1 { + t.Fatalf("Expected `x`.`y` to contain 1") + } + + q2 := query.NewMatchQuery("2") + q2.SetField("x.y") + if res, err := idx.Search(NewSearchRequest(q2)); err != nil || len(res.Hits) != 0 { + t.Fatalf("Expected `x`.`y` to not contain 2") + } + q2.SetField("`x.y`") + if res, err := idx.Search(NewSearchRequest(q2)); err != nil || len(res.Hits) != 1 { + t.Fatalf("Expected `x.y` to contain 2") + } +} diff --git a/test/tests/basic/searches.json b/test/tests/basic/searches.json index 7ddfce375..7678e4ae5 100644 --- a/test/tests/basic/searches.json +++ b/test/tests/basic/searches.json @@ -338,7 +338,7 @@ { "id": "a", "fields": { - "tags": ["gopher", "belieber"] + "`tags`": ["gopher", "belieber"] } } ] @@ -385,7 +385,7 @@ { "id": "b", "fragments": { - "name": ["steve has <a> long & complicated name"] + "`name`": ["steve has <a> long & complicated name"] } } ] @@ -409,7 +409,7 @@ { "id": "b", "fragments": { - "name": ["steve has <a> long & complicated name"] + "`name`": ["steve has <a> long & complicated name"] } } ] @@ -433,8 +433,8 @@ { "id": "b", "fields": { - "age": 27, - "birthday": "2001-09-09T01:46:40Z" + "`age`": 27, + "`birthday`": "2001-09-09T01:46:40Z" } } ] @@ -485,8 +485,8 @@ { "id": "b", "fragments": { - "name": ["steve has <a> long & complicated name"], - "title": ["missess"] + "`name`": ["steve has <a> long & complicated name"], + "`title`": ["missess"] } } ] @@ -512,7 +512,7 @@ { "id": "a", "fragments": { - "tags": ["gopher"] + "`tags`": ["gopher"] } } ] diff --git a/test/tests/employee/searches.json b/test/tests/employee/searches.json index d4db280b5..5e64cca60 100644 --- a/test/tests/employee/searches.json +++ b/test/tests/employee/searches.json @@ -17,7 +17,7 @@ { "id": "emp10508560", "locations": { - "manages.reports": { + "`manages`.`reports`": { "julián": [ { "pos": 2, @@ -38,4 +38,4 @@ ] } } -] \ No newline at end of file +] diff --git a/test/tests/facet/searches.json b/test/tests/facet/searches.json index 6752282a4..0d3bbe352 100644 --- a/test/tests/facet/searches.json +++ b/test/tests/facet/searches.json @@ -19,7 +19,7 @@ "hits": [], "facets": { "types": { - "field": "type", + "field": "`type`", "total": 10, "missing": 0, "other": 0, @@ -71,7 +71,7 @@ "hits": [], "facets": { "types": { - "field": "rating", + "field": "`rating`", "total": 10, "missing": 0, "other": 0, @@ -121,7 +121,7 @@ "hits": [], "facets": { "types": { - "field": "updated", + "field": "`updated`", "total": 10, "missing": 0, "other": 0, @@ -141,4 +141,4 @@ } } } -] \ No newline at end of file +] diff --git a/mapping/reflect.go b/util/reflect.go similarity index 50% rename from mapping/reflect.go rename to util/reflect.go index 6500a7059..f251d6f06 100644 --- a/mapping/reflect.go +++ b/util/reflect.go @@ -12,15 +12,81 @@ // See the License for the specific language governing permissions and // limitations under the License. -package mapping +package util import ( "reflect" "strings" ) +func LookupPropertyPathStr(data interface{}, path string) (string, bool) { + return mustString(lookupPropertyPath(data, path)) +} + +// ParseTagName extracts the field name from a struct tag +func ParseTagName(tag string) string { + if idx := strings.Index(tag, ","); idx != -1 { + return tag[:idx] + } + return tag +} + +// DecodePath splits a path into its parts +// For example: +// (1) a.b.c will be split into a, b and c +// (2) a.`b.c` will be split into a and b.c +// (3) `a.b`.c will be split into a.b and c +// (4) `a`.`b`.c will be split into a, b and c +func DecodePath(path string) []string { + var parts []string + var start int + var inQuote bool + for i, c := range path { + if c == '`' { + inQuote = !inQuote + } else if c == '.' && !inQuote { + parts = append(parts, stripEnclosingBackticks(path[start:i])) + start = i + 1 + } + } + parts = append(parts, stripEnclosingBackticks(path[start:])) + return parts +} + +// EncodePath concats a list of strings into a path +// by individually enclosing each string in backticks +// and separating them with a dot. +func EncodePath(pathElements []string) string { + var rv string + for i := 0; i < len(pathElements); i++ { + rv += encloseInBackticks(pathElements[i]) + if i < len(pathElements)-1 { + rv += pathSeparator + } + } + + return rv +} + +var internalFields = map[string]bool{ + "_all": true, + "_id": true, + "_score": true, +} + +// CleansePath cleanses a path by decoding and re-encoding it +// to make sure it is in the right format. +func CleansePath(path string) string { + if len(path) == 0 || internalFields[path] { + return path + } + return EncodePath(DecodePath(path)) +} + +// ----------------------------------------------------------------------------- + func lookupPropertyPath(data interface{}, path string) interface{} { - pathParts := decodePath(path) + pathParts := DecodePath(path) current := data for _, part := range pathParts { @@ -65,12 +131,15 @@ func lookupPropertyPathPart(data interface{}, part string) interface{} { const pathSeparator = "." -func decodePath(path string) []string { - return strings.Split(path, pathSeparator) +func encloseInBackticks(s string) string { + return "`" + s + "`" } -func encodePath(pathElements []string) string { - return strings.Join(pathElements, pathSeparator) +func stripEnclosingBackticks(s string) string { + if len(s) > 1 && s[0] == '`' && s[len(s)-1] == '`' { + return s[1 : len(s)-1] + } + return s } func mustString(data interface{}) (string, bool) { @@ -82,11 +151,3 @@ func mustString(data interface{}) (string, bool) { } return "", false } - -// parseTagName extracts the field name from a struct tag -func parseTagName(tag string) string { - if idx := strings.Index(tag, ","); idx != -1 { - return tag[:idx] - } - return tag -} diff --git a/mapping/reflect_test.go b/util/reflect_test.go similarity index 97% rename from mapping/reflect_test.go rename to util/reflect_test.go index fdba06f88..d77dcc02a 100644 --- a/mapping/reflect_test.go +++ b/util/reflect_test.go @@ -1,4 +1,4 @@ -package mapping +package util import ( "reflect"