diff --git a/docs/dynamic-templates.md b/docs/dynamic-templates.md new file mode 100644 index 000000000..da262408b --- /dev/null +++ b/docs/dynamic-templates.md @@ -0,0 +1,300 @@ +# Dynamic Templates + +Dynamic templates allow you to define rules for how dynamically detected fields are mapped. When a document contains fields that don't have explicit mappings and dynamic mapping is enabled, templates are evaluated in order to determine how those fields should be indexed. + +This feature is inspired by [Elasticsearch's dynamic_templates](https://www.elastic.co/guide/en/elasticsearch/reference/current/dynamic-templates.html) functionality. + +## Overview + +When indexing documents with dynamic mapping enabled, bleve automatically detects field types and creates appropriate field mappings. Dynamic templates give you fine-grained control over this process by letting you: + +- Apply specific analyzers to fields matching name patterns +- Control which fields are stored, indexed, or have doc values +- Override the default type detection +- Exclude certain fields from specific mappings + +## Template Definition + +A dynamic template consists of matching criteria and a field mapping to apply: + +```go +type DynamicTemplate struct { + Name string // Optional identifier for debugging + Match string // Glob pattern for field name + Unmatch string // Exclusion pattern for field name + PathMatch string // Glob pattern for full field path + PathUnmatch string // Exclusion pattern for full path + MatchMappingType string // Filter by detected type + Mapping *FieldMapping // Mapping to apply when matched +} +``` + +### Matching Criteria + +Templates support multiple matching criteria that are evaluated together (all specified criteria must match): + +| Field | Description | Example | +|-------|-------------|---------| +| `Match` | Glob pattern for the field name (last path element) | `*_text`, `field_*` | +| `Unmatch` | Exclusion pattern for field name | `skip_*` | +| `PathMatch` | Glob pattern for the full dotted path | `metadata.**`, `user.*.name` | +| `PathUnmatch` | Exclusion pattern for full path | `internal.**` | +| `MatchMappingType` | Filter by detected type | `string`, `number`, `boolean`, `date` | + +### Glob Pattern Syntax + +Patterns use the [doublestar](https://github.com/bmatcuk/doublestar) library for matching: + +- `*` - matches any sequence of characters within a single path segment +- `**` - matches any sequence of characters across multiple path segments + +Examples: +- `*_text` matches `title_text`, `body_text` +- `field_*` matches `field_name`, `field_value` +- `metadata.**` matches `metadata.author`, `metadata.tags.primary` +- `*.name` matches `user.name`, `product.name` + +### Detected Types + +The `MatchMappingType` field accepts these values: + +| Type | Description | +|------|-------------| +| `string` | String values | +| `number` | Numeric values (int, float) | +| `boolean` | Boolean values | +| `date` | time.Time values | +| `object` | Nested objects/maps | + +## Usage + +### Go API + +```go +import "github.com/blevesearch/bleve/v2" + +// Create an index mapping +mapping := bleve.NewIndexMapping() + +// Add a template for keyword fields +mapping.DefaultMapping.AddDynamicTemplate( + mapping.NewDynamicTemplate("keyword_fields"). + MatchField("*_keyword"). + MatchType("string"). + WithMapping(&mapping.FieldMapping{ + Type: "text", + Analyzer: "keyword", + }), +) + +// Add a template for text fields with English analyzer +mapping.DefaultMapping.AddDynamicTemplate( + mapping.NewDynamicTemplate("english_text"). + MatchField("*_text"). + MatchType("string"). + WithMapping(&mapping.FieldMapping{ + Type: "text", + Analyzer: "en", + }), +) + +// Add a template for all strings under metadata path +mapping.DefaultMapping.AddDynamicTemplate( + mapping.NewDynamicTemplate("metadata_strings"). + MatchPath("metadata.**"). + MatchType("string"). + WithMapping(&mapping.FieldMapping{ + Type: "text", + Analyzer: "keyword", + Store: true, + }), +) +``` + +### JSON Configuration + +```json +{ + "default_mapping": { + "enabled": true, + "dynamic": true, + "dynamic_templates": [ + { + "name": "keyword_fields", + "match": "*_keyword", + "match_mapping_type": "string", + "mapping": { + "type": "text", + "analyzer": "keyword" + } + }, + { + "name": "english_text", + "match": "*_text", + "match_mapping_type": "string", + "mapping": { + "type": "text", + "analyzer": "en" + } + }, + { + "name": "metadata_strings", + "path_match": "metadata.**", + "match_mapping_type": "string", + "mapping": { + "type": "text", + "analyzer": "keyword", + "store": true + } + } + ] + } +} +``` + +## Template Inheritance + +Templates are inherited through the document mapping hierarchy: + +1. Templates defined at parent mappings are inherited by child mappings +2. Child mappings can define their own templates that take precedence +3. Templates at the closest mapping level are checked first +4. The first matching template wins + +```go +// Root-level template +mapping.DefaultMapping.AddDynamicTemplate( + mapping.NewDynamicTemplate("all_strings"). + MatchType("string"). + WithMapping(&mapping.FieldMapping{ + Type: "text", + Analyzer: "standard", + }), +) + +// Sub-document specific template (takes precedence for fields under "logs") +logsMapping := bleve.NewDocumentMapping() +logsMapping.AddDynamicTemplate( + mapping.NewDynamicTemplate("log_strings"). + MatchType("string"). + WithMapping(&mapping.FieldMapping{ + Type: "text", + Analyzer: "keyword", // logs use keyword analyzer + }), +) +mapping.DefaultMapping.AddSubDocumentMapping("logs", logsMapping) +``` + +## Template Evaluation Order + +1. When a dynamic field is encountered, templates at the closest document mapping are checked first +2. If no match is found, parent templates are checked (inherited templates) +3. Templates within each level are evaluated in the order they were added +4. The first template whose criteria all match is used +5. If no template matches, default dynamic mapping behavior applies + +## Examples + +### Index Log Levels as Keywords + +```go +mapping.DefaultMapping.AddDynamicTemplate( + mapping.NewDynamicTemplate("log_levels"). + MatchField("level"). + MatchType("string"). + WithMapping(&mapping.FieldMapping{ + Type: "text", + Analyzer: "keyword", + DocValues: true, // Enable for faceting + }), +) +``` + +### Exclude Internal Fields from Full-Text Search + +```go +mapping.DefaultMapping.AddDynamicTemplate( + mapping.NewDynamicTemplate("internal_fields"). + MatchPath("_internal.**"). + WithMapping(&mapping.FieldMapping{ + Type: "text", + Index: false, // Don't index these fields + Store: true, // But still store them + }), +) +``` + +### Different Analyzers for Different Languages + +```go +// English content +mapping.DefaultMapping.AddDynamicTemplate( + mapping.NewDynamicTemplate("english_content"). + MatchPath("content.en.**"). + MatchType("string"). + WithMapping(&mapping.FieldMapping{ + Type: "text", + Analyzer: "en", + }), +) + +// German content +mapping.DefaultMapping.AddDynamicTemplate( + mapping.NewDynamicTemplate("german_content"). + MatchPath("content.de.**"). + MatchType("string"). + WithMapping(&mapping.FieldMapping{ + Type: "text", + Analyzer: "de", + }), +) +``` + +### Combine Name and Type Matching + +```go +// Only match numeric fields ending in "_count" +mapping.DefaultMapping.AddDynamicTemplate( + mapping.NewDynamicTemplate("count_fields"). + MatchField("*_count"). + MatchType("number"). + WithMapping(&mapping.FieldMapping{ + Type: "number", + DocValues: true, + Store: true, + }), +) +``` + +## Default Behavior + +When a template matches but doesn't explicitly set `Store`, `Index`, or `DocValues`, the global dynamic settings are applied: + +- `IndexDynamic` - whether to index dynamic fields (default: true) +- `StoreDynamic` - whether to store dynamic fields (default: true) +- `DocValuesDynamic` - whether to enable doc values for dynamic fields (default: true) + +These can be configured at the `IndexMapping` level: + +```go +mapping := bleve.NewIndexMapping() +mapping.StoreDynamic = false // Don't store dynamic fields by default +mapping.IndexDynamic = true // But still index them +mapping.DocValuesDynamic = true // Enable doc values for sorting/faceting +``` + +## Strict JSON Validation + +When `mapping.MappingJSONStrict = true`, invalid keys in template JSON will cause an error: + +```go +mapping.MappingJSONStrict = true + +// This will error due to "invalid_key" +jsonData := `{ + "name": "test", + "invalid_key": "value" +}` +var template mapping.DynamicTemplate +err := json.Unmarshal([]byte(jsonData), &template) // Returns error +``` diff --git a/go.mod b/go.mod index 2800d13ca..7bddf4e79 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/blevesearch/zapx/v15 v15.4.2 github.com/blevesearch/zapx/v16 v16.3.0 github.com/blevesearch/zapx/v17 v17.0.0 + github.com/bmatcuk/doublestar/v4 v4.9.2 github.com/couchbase/moss v0.2.0 github.com/spf13/cobra v1.8.1 go.etcd.io/bbolt v1.4.0 diff --git a/go.sum b/go.sum index c59e03121..b7938d50b 100644 --- a/go.sum +++ b/go.sum @@ -48,6 +48,8 @@ github.com/blevesearch/zapx/v16 v16.3.0 h1:hF6VlN15E9CB40RMPyqOIhlDw1OOo9RItumhK github.com/blevesearch/zapx/v16 v16.3.0/go.mod h1:zCFjv7McXWm1C8rROL+3mUoD5WYe2RKsZP3ufqcYpLY= github.com/blevesearch/zapx/v17 v17.0.0 h1:srLJFkv5ghz1Z8iVz5uoOK89G2NvI4KdMG7aF3Cx7rE= github.com/blevesearch/zapx/v17 v17.0.0/go.mod h1:/pi9Gq7byQcduhNB6Vk08+ZXGVGPjZoNc5QnQY8lkOo= +github.com/bmatcuk/doublestar/v4 v4.9.2 h1:b0mc6WyRSYLjzofB2v/0cuDUZ+MqoGyH3r0dVij35GI= +github.com/bmatcuk/doublestar/v4 v4.9.2/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/couchbase/ghistogram v0.1.0 h1:b95QcQTCzjTUocDXp/uMgSNQi8oj1tGwnJ4bODWZnps= github.com/couchbase/ghistogram v0.1.0/go.mod h1:s1Jhy76zqfEecpNWJfWUiKZookAFaiGOEoyzgHt9i7k= github.com/couchbase/moss v0.2.0 h1:VCYrMzFwEryyhRSeI+/b3tRBSeTpi/8gn5Kf6dxqn+o= diff --git a/mapping/document.go b/mapping/document.go index 3da925038..499827bce 100644 --- a/mapping/document.go +++ b/mapping/document.go @@ -49,6 +49,11 @@ type DocumentMapping struct { DefaultAnalyzer string `json:"default_analyzer,omitempty"` DefaultSynonymSource string `json:"default_synonym_source,omitempty"` + // DynamicTemplates define rules for mapping dynamically detected fields. + // Templates are evaluated in order; the first matching template is used. + // Templates are inherited from parent mappings but can be overridden. + DynamicTemplates []*DynamicTemplate `json:"dynamic_templates,omitempty"` + // StructTagKey overrides "json" when looking for field names in struct tags StructTagKey string `json:"struct_tag_key,omitempty"` } @@ -311,6 +316,72 @@ func (dm *DocumentMapping) AddFieldMapping(fm *FieldMapping) { dm.Fields = append(dm.Fields, fm) } +// AddDynamicTemplate adds a dynamic template to this document mapping. +// Templates are evaluated in order when mapping dynamic fields. +func (dm *DocumentMapping) AddDynamicTemplate(template *DynamicTemplate) { + if dm.DynamicTemplates == nil { + dm.DynamicTemplates = make([]*DynamicTemplate, 0) + } + dm.DynamicTemplates = append(dm.DynamicTemplates, template) +} + +// findMatchingTemplate searches for a matching dynamic template for a field. +// It checks templates at the current mapping level first, then inherits from +// parent templates if no local match is found. +// Parameters: +// - fieldName: the name of the field (last path element) +// - pathStr: the full dotted path to the field +// - detectedType: the detected type of the field value ("string", "number", etc.) +// - parentTemplates: templates inherited from parent document mappings +func (dm *DocumentMapping) findMatchingTemplate(fieldName, pathStr, detectedType string, + parentTemplates []*DynamicTemplate) *DynamicTemplate { + // Check own templates first (local templates take precedence) + for _, template := range dm.DynamicTemplates { + if template.Matches(fieldName, pathStr, detectedType) { + return template + } + } + + // Fall back to parent templates (inheritance) + for _, template := range parentTemplates { + if template.Matches(fieldName, pathStr, detectedType) { + return template + } + } + + return nil +} + +// collectTemplatesAlongPath collects all dynamic templates along the path to a field. +// Templates from parent mappings are collected first, so child mappings can override. +// This collects templates from all ancestor mappings but NOT the closest mapping +// (which is checked separately in findMatchingTemplate). +func (dm *DocumentMapping) collectTemplatesAlongPath(pathElements []string) []*DynamicTemplate { + var templates []*DynamicTemplate + + // Start with templates at the root + templates = append(templates, dm.DynamicTemplates...) + + // Walk down the path, collecting templates from each level EXCEPT the closest one. + // For path ["a", "b", "field"], we want templates from root and "a", but not "b" + // because "b" is the closest mapping and will be checked separately. + current := dm + stopAt := max(0, len(pathElements)-2) + for i, pathElement := range pathElements { + if i >= stopAt { + break + } + if subDocMapping, exists := current.Properties[pathElement]; exists { + templates = append(templates, subDocMapping.DynamicTemplates...) + current = subDocMapping + } else { + break + } + } + + return templates +} + // UnmarshalJSON offers custom unmarshaling with optional strict validation func (dm *DocumentMapping) UnmarshalJSON(data []byte) error { var tmp map[string]json.RawMessage @@ -366,6 +437,11 @@ func (dm *DocumentMapping) UnmarshalJSON(data []byte) error { if err != nil { return err } + case "dynamic_templates": + err := util.UnmarshalJSON(v, &dm.DynamicTemplates) + if err != nil { + return err + } default: invalidKeys = append(invalidKeys, k) } @@ -564,18 +640,43 @@ func (dm *DocumentMapping) processProperty(property interface{}, path []string, } else if closestDocMapping.Dynamic { // automatic indexing behavior - // first see if it can be parsed by the default date parser - dateTimeParser := context.im.DateTimeParserNamed(context.im.DefaultDateTimeParser) - if dateTimeParser != nil { - parsedDateTime, layout, err := dateTimeParser.ParseDateTime(propertyValueString) - if err != nil { - // index as text - fieldMapping := newTextFieldMappingDynamic(context.im) + // Check for a matching dynamic template first + fieldName := "" + if len(path) > 0 { + fieldName = path[len(path)-1] + } + parentTemplates := dm.collectTemplatesAlongPath(path) + template := closestDocMapping.findMatchingTemplate(fieldName, pathString, "string", parentTemplates) + + if template != nil && template.Mapping != nil { + // Use the template's mapping with dynamic defaults + fieldMapping := applyDynamicDefaults(template.Mapping, context.im) + switch fieldMapping.Type { + case "datetime": + dateTimeParser := context.im.DateTimeParserNamed(context.im.DefaultDateTimeParser) + if dateTimeParser != nil { + parsedDateTime, layout, err := dateTimeParser.ParseDateTime(propertyValueString) + if err == nil { + fieldMapping.processTime(parsedDateTime, layout, pathString, path, indexes, context) + } + } + default: fieldMapping.processString(propertyValueString, pathString, path, indexes, context) - } else { - // index as datetime - fieldMapping := newDateTimeFieldMappingDynamic(context.im) - fieldMapping.processTime(parsedDateTime, layout, pathString, path, indexes, context) + } + } else { + // Default behavior: first see if it can be parsed by the default date parser + dateTimeParser := context.im.DateTimeParserNamed(context.im.DefaultDateTimeParser) + if dateTimeParser != nil { + parsedDateTime, layout, err := dateTimeParser.ParseDateTime(propertyValueString) + if err != nil { + // index as text + fieldMapping := newTextFieldMappingDynamic(context.im) + fieldMapping.processString(propertyValueString, pathString, path, indexes, context) + } else { + // index as datetime + fieldMapping := newDateTimeFieldMappingDynamic(context.im) + fieldMapping.processTime(parsedDateTime, layout, pathString, path, indexes, context) + } } } } @@ -593,9 +694,23 @@ func (dm *DocumentMapping) processProperty(property interface{}, path []string, fieldMapping.processFloat64(propertyValFloat, pathString, path, indexes, context) } } else if closestDocMapping.Dynamic { - // automatic indexing behavior - fieldMapping := newNumericFieldMappingDynamic(context.im) - fieldMapping.processFloat64(propertyValFloat, pathString, path, indexes, context) + // Check for a matching dynamic template first + fieldName := "" + if len(path) > 0 { + fieldName = path[len(path)-1] + } + parentTemplates := dm.collectTemplatesAlongPath(path) + template := closestDocMapping.findMatchingTemplate(fieldName, pathString, "number", parentTemplates) + + if template != nil && template.Mapping != nil { + // Use the template's mapping with dynamic defaults + fieldMapping := applyDynamicDefaults(template.Mapping, context.im) + fieldMapping.processFloat64(propertyValFloat, pathString, path, indexes, context) + } else { + // automatic indexing behavior + fieldMapping := newNumericFieldMappingDynamic(context.im) + fieldMapping.processFloat64(propertyValFloat, pathString, path, indexes, context) + } } case reflect.Bool: propertyValBool := propertyValue.Bool() @@ -605,9 +720,23 @@ func (dm *DocumentMapping) processProperty(property interface{}, path []string, fieldMapping.processBoolean(propertyValBool, pathString, path, indexes, context) } } else if closestDocMapping.Dynamic { - // automatic indexing behavior - fieldMapping := newBooleanFieldMappingDynamic(context.im) - fieldMapping.processBoolean(propertyValBool, pathString, path, indexes, context) + // Check for a matching dynamic template first + fieldName := "" + if len(path) > 0 { + fieldName = path[len(path)-1] + } + parentTemplates := dm.collectTemplatesAlongPath(path) + template := closestDocMapping.findMatchingTemplate(fieldName, pathString, "boolean", parentTemplates) + + if template != nil && template.Mapping != nil { + // Use the template's mapping with dynamic defaults + fieldMapping := applyDynamicDefaults(template.Mapping, context.im) + fieldMapping.processBoolean(propertyValBool, pathString, path, indexes, context) + } else { + // automatic indexing behavior + fieldMapping := newBooleanFieldMappingDynamic(context.im) + fieldMapping.processBoolean(propertyValBool, pathString, path, indexes, context) + } } case reflect.Struct: switch property := property.(type) { @@ -619,8 +748,22 @@ func (dm *DocumentMapping) processProperty(property interface{}, path []string, fieldMapping.processTime(property, time.RFC3339, pathString, path, indexes, context) } } else if closestDocMapping.Dynamic { - fieldMapping := newDateTimeFieldMappingDynamic(context.im) - fieldMapping.processTime(property, time.RFC3339, pathString, path, indexes, context) + // Check for a matching dynamic template first + fieldName := "" + if len(path) > 0 { + fieldName = path[len(path)-1] + } + parentTemplates := dm.collectTemplatesAlongPath(path) + template := closestDocMapping.findMatchingTemplate(fieldName, pathString, "date", parentTemplates) + + if template != nil && template.Mapping != nil { + // Use the template's mapping with dynamic defaults + fieldMapping := applyDynamicDefaults(template.Mapping, context.im) + fieldMapping.processTime(property, time.RFC3339, pathString, path, indexes, context) + } else { + fieldMapping := newDateTimeFieldMappingDynamic(context.im) + fieldMapping.processTime(property, time.RFC3339, pathString, path, indexes, context) + } } case encoding.TextMarshaler: txt, err := property.MarshalText() diff --git a/mapping/dynamic_template.go b/mapping/dynamic_template.go new file mode 100644 index 000000000..0e8962a36 --- /dev/null +++ b/mapping/dynamic_template.go @@ -0,0 +1,238 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mapping + +import ( + "encoding/json" + "fmt" + "reflect" + "time" + + "github.com/blevesearch/bleve/v2/util" + "github.com/bmatcuk/doublestar/v4" +) + +// DynamicTemplate defines a rule for mapping dynamically detected fields. +// When a field is encountered that has no explicit mapping and dynamic mapping +// is enabled, templates are checked in order. The first matching template's +// mapping is used for the field. +// +// This is similar to Elasticsearch's dynamic_templates feature. +type DynamicTemplate struct { + // Name is an optional identifier for this template (useful for debugging) + Name string `json:"name,omitempty"` + + // Match is a glob pattern to match against the field name (last path element). + // Supports * and ** wildcards via doublestar library. + // Example: "*_text" matches "title_text", "body_text" + Match string `json:"match,omitempty"` + + // Unmatch is a glob pattern; if it matches the field name, the template is skipped. + // Example: "skip_*" would exclude fields like "skip_this" + Unmatch string `json:"unmatch,omitempty"` + + // PathMatch is a glob pattern to match against the full dotted path. + // Supports ** for matching multiple path segments. + // Example: "metadata.**" matches "metadata.author", "metadata.tags.primary" + PathMatch string `json:"path_match,omitempty"` + + // PathUnmatch is a glob pattern; if it matches the full path, the template is skipped. + PathUnmatch string `json:"path_unmatch,omitempty"` + + // MatchMappingType filters by the detected data type. + // Valid values: "string", "number", "boolean", "date", "object" + MatchMappingType string `json:"match_mapping_type,omitempty"` + + // Mapping is the field mapping to apply when this template matches. + Mapping *FieldMapping `json:"mapping,omitempty"` +} + +// NewDynamicTemplate creates a new DynamicTemplate with the given name. +func NewDynamicTemplate(name string) *DynamicTemplate { + return &DynamicTemplate{Name: name} +} + +// MatchField sets the field name pattern to match. +func (t *DynamicTemplate) MatchField(pattern string) *DynamicTemplate { + t.Match = pattern + return t +} + +// UnmatchField sets the field name pattern to exclude. +func (t *DynamicTemplate) UnmatchField(pattern string) *DynamicTemplate { + t.Unmatch = pattern + return t +} + +// MatchPath sets the path pattern to match. +func (t *DynamicTemplate) MatchPath(pattern string) *DynamicTemplate { + t.PathMatch = pattern + return t +} + +// UnmatchPath sets the path pattern to exclude. +func (t *DynamicTemplate) UnmatchPath(pattern string) *DynamicTemplate { + t.PathUnmatch = pattern + return t +} + +// MatchType sets the mapping type to match. +func (t *DynamicTemplate) MatchType(typeName string) *DynamicTemplate { + t.MatchMappingType = typeName + return t +} + +// WithMapping sets the field mapping to use when this template matches. +func (t *DynamicTemplate) WithMapping(m *FieldMapping) *DynamicTemplate { + t.Mapping = m + return t +} + +// Matches checks if this template matches the given field. +// All specified criteria must match for the template to be considered a match. +func (t *DynamicTemplate) Matches(fieldName, pathStr, detectedType string) bool { + // Check match_mapping_type first (most selective usually) + if t.MatchMappingType != "" && t.MatchMappingType != detectedType { + return false + } + + // Check field name match pattern + if t.Match != "" { + matched, err := doublestar.Match(t.Match, fieldName) + if err != nil || !matched { + return false + } + } + + // Check field name unmatch pattern (exclusion) + if t.Unmatch != "" { + matched, err := doublestar.Match(t.Unmatch, fieldName) + if err == nil && matched { + return false + } + } + + // Check path match pattern + if t.PathMatch != "" { + matched, err := doublestar.Match(t.PathMatch, pathStr) + if err != nil || !matched { + return false + } + } + + // Check path unmatch pattern (exclusion) + if t.PathUnmatch != "" { + matched, err := doublestar.Match(t.PathUnmatch, pathStr) + if err == nil && matched { + return false + } + } + + return true +} + +// detectMappingType returns the bleve mapping type name for a reflected value. +// This is used to match against MatchMappingType in dynamic templates. +func detectMappingType(val reflect.Value) string { + if !val.IsValid() { + return "" + } + + switch val.Kind() { + case reflect.String: + return "string" + case reflect.Float32, reflect.Float64, + reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return "number" + case reflect.Bool: + return "boolean" + case reflect.Struct: + // Check if it's a time.Time + if val.Type() == reflect.TypeOf(time.Time{}) { + return "date" + } + return "object" + case reflect.Map: + return "object" + case reflect.Slice, reflect.Array: + return "array" + case reflect.Ptr: + if !val.IsNil() { + return detectMappingType(val.Elem()) + } + return "" + default: + return "" + } +} + +// UnmarshalJSON offers custom unmarshaling with optional strict validation +func (t *DynamicTemplate) UnmarshalJSON(data []byte) error { + var tmp map[string]json.RawMessage + err := util.UnmarshalJSON(data, &tmp) + if err != nil { + return err + } + + var invalidKeys []string + for k, v := range tmp { + switch k { + case "name": + err := util.UnmarshalJSON(v, &t.Name) + if err != nil { + return err + } + case "match": + err := util.UnmarshalJSON(v, &t.Match) + if err != nil { + return err + } + case "unmatch": + err := util.UnmarshalJSON(v, &t.Unmatch) + if err != nil { + return err + } + case "path_match": + err := util.UnmarshalJSON(v, &t.PathMatch) + if err != nil { + return err + } + case "path_unmatch": + err := util.UnmarshalJSON(v, &t.PathUnmatch) + if err != nil { + return err + } + case "match_mapping_type": + err := util.UnmarshalJSON(v, &t.MatchMappingType) + if err != nil { + return err + } + case "mapping": + err := util.UnmarshalJSON(v, &t.Mapping) + if err != nil { + return err + } + default: + invalidKeys = append(invalidKeys, k) + } + } + + if MappingJSONStrict && len(invalidKeys) > 0 { + return fmt.Errorf("dynamic template contains invalid keys: %v", invalidKeys) + } + + return nil +} diff --git a/mapping/dynamic_template_test.go b/mapping/dynamic_template_test.go new file mode 100644 index 000000000..ff494cdde --- /dev/null +++ b/mapping/dynamic_template_test.go @@ -0,0 +1,813 @@ +// Copyright (c) 2014 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mapping + +import ( + "encoding/json" + "reflect" + "testing" + "time" + + "github.com/blevesearch/bleve/v2/document" +) + +func TestDynamicTemplateMatching(t *testing.T) { + tests := []struct { + name string + template *DynamicTemplate + fieldName string + pathStr string + detectedType string + shouldMatch bool + }{ + { + name: "match field name with wildcard suffix", + template: &DynamicTemplate{ + Match: "*_text", + }, + fieldName: "title_text", + pathStr: "title_text", + detectedType: "string", + shouldMatch: true, + }, + { + name: "match field name with wildcard prefix", + template: &DynamicTemplate{ + Match: "field_*", + }, + fieldName: "field_name", + pathStr: "field_name", + detectedType: "string", + shouldMatch: true, + }, + { + name: "no match field name", + template: &DynamicTemplate{ + Match: "*_text", + }, + fieldName: "title_keyword", + pathStr: "title_keyword", + detectedType: "string", + shouldMatch: false, + }, + { + name: "match path with double star", + template: &DynamicTemplate{ + PathMatch: "metadata.**", + }, + fieldName: "author", + pathStr: "metadata.author", + detectedType: "string", + shouldMatch: true, + }, + { + name: "match nested path with double star", + template: &DynamicTemplate{ + PathMatch: "metadata.**", + }, + fieldName: "primary", + pathStr: "metadata.tags.primary", + detectedType: "string", + shouldMatch: true, + }, + { + name: "no match path", + template: &DynamicTemplate{ + PathMatch: "metadata.**", + }, + fieldName: "name", + pathStr: "content.name", + detectedType: "string", + shouldMatch: false, + }, + { + name: "match mapping type", + template: &DynamicTemplate{ + MatchMappingType: "string", + }, + fieldName: "anything", + pathStr: "anything", + detectedType: "string", + shouldMatch: true, + }, + { + name: "no match mapping type", + template: &DynamicTemplate{ + MatchMappingType: "number", + }, + fieldName: "anything", + pathStr: "anything", + detectedType: "string", + shouldMatch: false, + }, + { + name: "match with unmatch exclusion", + template: &DynamicTemplate{ + Match: "*_field", + Unmatch: "skip_*", + }, + fieldName: "title_field", + pathStr: "title_field", + detectedType: "string", + shouldMatch: true, + }, + { + name: "excluded by unmatch", + template: &DynamicTemplate{ + Match: "*_field", + Unmatch: "skip_*", + }, + fieldName: "skip_field", + pathStr: "skip_field", + detectedType: "string", + shouldMatch: false, + }, + { + name: "match with path_unmatch exclusion", + template: &DynamicTemplate{ + PathMatch: "data.**", + PathUnmatch: "data.internal.**", + }, + fieldName: "value", + pathStr: "data.public.value", + detectedType: "string", + shouldMatch: true, + }, + { + name: "excluded by path_unmatch", + template: &DynamicTemplate{ + PathMatch: "data.**", + PathUnmatch: "data.internal.**", + }, + fieldName: "secret", + pathStr: "data.internal.secret", + detectedType: "string", + shouldMatch: false, + }, + { + name: "combined match criteria", + template: &DynamicTemplate{ + Match: "*_count", + MatchMappingType: "number", + }, + fieldName: "item_count", + pathStr: "item_count", + detectedType: "number", + shouldMatch: true, + }, + { + name: "combined match - type mismatch", + template: &DynamicTemplate{ + Match: "*_count", + MatchMappingType: "number", + }, + fieldName: "item_count", + pathStr: "item_count", + detectedType: "string", // type doesn't match + shouldMatch: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := tt.template.Matches(tt.fieldName, tt.pathStr, tt.detectedType) + if result != tt.shouldMatch { + t.Errorf("expected match=%v, got %v", tt.shouldMatch, result) + } + }) + } +} + +func TestDetectMappingType(t *testing.T) { + tests := []struct { + name string + value interface{} + expected string + }{ + {"string", "hello", "string"}, + {"int", 42, "number"}, + {"int64", int64(42), "number"}, + {"float64", 3.14, "number"}, + {"bool", true, "boolean"}, + {"time", time.Now(), "date"}, + {"map", map[string]interface{}{}, "object"}, + {"slice", []string{}, "array"}, + {"nil pointer", (*string)(nil), ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + val := reflect.ValueOf(tt.value) + result := detectMappingType(val) + if result != tt.expected { + t.Errorf("expected type=%s, got %s", tt.expected, result) + } + }) + } +} + +func TestDynamicTemplateJSON(t *testing.T) { + jsonData := `{ + "name": "keyword_fields", + "match": "*_keyword", + "unmatch": "skip_*", + "path_match": "data.**", + "path_unmatch": "data.internal.**", + "match_mapping_type": "string", + "mapping": { + "type": "text", + "analyzer": "keyword", + "store": true, + "index": true + } + }` + + var template DynamicTemplate + err := json.Unmarshal([]byte(jsonData), &template) + if err != nil { + t.Fatalf("failed to unmarshal: %v", err) + } + + if template.Name != "keyword_fields" { + t.Errorf("expected name=keyword_fields, got %s", template.Name) + } + if template.Match != "*_keyword" { + t.Errorf("expected match=*_keyword, got %s", template.Match) + } + if template.Unmatch != "skip_*" { + t.Errorf("expected unmatch=skip_*, got %s", template.Unmatch) + } + if template.PathMatch != "data.**" { + t.Errorf("expected path_match=data.**, got %s", template.PathMatch) + } + if template.PathUnmatch != "data.internal.**" { + t.Errorf("expected path_unmatch=data.internal.**, got %s", template.PathUnmatch) + } + if template.MatchMappingType != "string" { + t.Errorf("expected match_mapping_type=string, got %s", template.MatchMappingType) + } + if template.Mapping == nil { + t.Fatal("expected mapping to be set") + } + if template.Mapping.Type != "text" { + t.Errorf("expected mapping.type=text, got %s", template.Mapping.Type) + } + if template.Mapping.Analyzer != "keyword" { + t.Errorf("expected mapping.analyzer=keyword, got %s", template.Mapping.Analyzer) + } +} + +func TestDynamicTemplateJSONStrict(t *testing.T) { + MappingJSONStrict = true + defer func() { + MappingJSONStrict = false + }() + + jsonData := `{ + "name": "test", + "invalid_key": "value" + }` + + var template DynamicTemplate + err := json.Unmarshal([]byte(jsonData), &template) + if err == nil { + t.Error("expected error for invalid key in strict mode") + } +} + +func TestDocumentMappingWithDynamicTemplates(t *testing.T) { + jsonData := `{ + "enabled": true, + "dynamic": true, + "dynamic_templates": [ + { + "name": "text_fields", + "match": "*_text", + "mapping": { + "type": "text", + "analyzer": "en" + } + }, + { + "name": "keyword_fields", + "match": "*_keyword", + "mapping": { + "type": "text", + "analyzer": "keyword" + } + } + ] + }` + + var dm DocumentMapping + err := json.Unmarshal([]byte(jsonData), &dm) + if err != nil { + t.Fatalf("failed to unmarshal: %v", err) + } + + if len(dm.DynamicTemplates) != 2 { + t.Fatalf("expected 2 templates, got %d", len(dm.DynamicTemplates)) + } + + if dm.DynamicTemplates[0].Name != "text_fields" { + t.Errorf("expected first template name=text_fields, got %s", dm.DynamicTemplates[0].Name) + } + if dm.DynamicTemplates[1].Name != "keyword_fields" { + t.Errorf("expected second template name=keyword_fields, got %s", dm.DynamicTemplates[1].Name) + } +} + +func TestDynamicTemplateFluentAPI(t *testing.T) { + template := NewDynamicTemplate("my_template"). + MatchField("*_text"). + UnmatchField("skip_*"). + MatchPath("data.**"). + UnmatchPath("data.internal.**"). + MatchType("string"). + WithMapping(NewTextFieldMapping()) + + if template.Name != "my_template" { + t.Errorf("expected name=my_template, got %s", template.Name) + } + if template.Match != "*_text" { + t.Errorf("expected match=*_text, got %s", template.Match) + } + if template.Unmatch != "skip_*" { + t.Errorf("expected unmatch=skip_*, got %s", template.Unmatch) + } + if template.PathMatch != "data.**" { + t.Errorf("expected path_match=data.**, got %s", template.PathMatch) + } + if template.PathUnmatch != "data.internal.**" { + t.Errorf("expected path_unmatch=data.internal.**, got %s", template.PathUnmatch) + } + if template.MatchMappingType != "string" { + t.Errorf("expected match_mapping_type=string, got %s", template.MatchMappingType) + } + if template.Mapping == nil { + t.Error("expected mapping to be set") + } +} + +func TestAddDynamicTemplate(t *testing.T) { + dm := NewDocumentMapping() + + template1 := NewDynamicTemplate("template1").MatchField("*_text") + template2 := NewDynamicTemplate("template2").MatchField("*_keyword") + + dm.AddDynamicTemplate(template1) + dm.AddDynamicTemplate(template2) + + if len(dm.DynamicTemplates) != 2 { + t.Fatalf("expected 2 templates, got %d", len(dm.DynamicTemplates)) + } +} + +func TestFindMatchingTemplate(t *testing.T) { + dm := NewDocumentMapping() + + textTemplate := NewDynamicTemplate("text_fields"). + MatchField("*_text"). + WithMapping(&FieldMapping{Type: "text", Analyzer: "en"}) + + keywordTemplate := NewDynamicTemplate("keyword_fields"). + MatchField("*_keyword"). + WithMapping(&FieldMapping{Type: "text", Analyzer: "keyword"}) + + dm.AddDynamicTemplate(textTemplate) + dm.AddDynamicTemplate(keywordTemplate) + + // Test finding the text template + found := dm.findMatchingTemplate("title_text", "title_text", "string", nil) + if found == nil { + t.Fatal("expected to find template") + } + if found.Name != "text_fields" { + t.Errorf("expected template name=text_fields, got %s", found.Name) + } + + // Test finding the keyword template + found = dm.findMatchingTemplate("status_keyword", "status_keyword", "string", nil) + if found == nil { + t.Fatal("expected to find template") + } + if found.Name != "keyword_fields" { + t.Errorf("expected template name=keyword_fields, got %s", found.Name) + } + + // Test no match + found = dm.findMatchingTemplate("other_field", "other_field", "string", nil) + if found != nil { + t.Error("expected no template match") + } +} + +func TestFindMatchingTemplateWithInheritance(t *testing.T) { + dm := NewDocumentMapping() + + // Parent template + parentTemplate := NewDynamicTemplate("parent_strings"). + MatchType("string"). + WithMapping(&FieldMapping{Type: "text", Analyzer: "standard"}) + dm.AddDynamicTemplate(parentTemplate) + + // Child mapping with its own template + childMapping := NewDocumentMapping() + childTemplate := NewDynamicTemplate("child_text"). + MatchField("*_text"). + WithMapping(&FieldMapping{Type: "text", Analyzer: "en"}) + childMapping.AddDynamicTemplate(childTemplate) + dm.AddSubDocumentMapping("child", childMapping) + + // Test that child template takes precedence for matching fields + parentTemplates := dm.DynamicTemplates + found := childMapping.findMatchingTemplate("title_text", "child.title_text", "string", parentTemplates) + if found == nil { + t.Fatal("expected to find template") + } + if found.Name != "child_text" { + t.Errorf("expected child template to match, got %s", found.Name) + } + + // Test that parent template is used when child doesn't match + found = childMapping.findMatchingTemplate("other_field", "child.other_field", "string", parentTemplates) + if found == nil { + t.Fatal("expected to find parent template") + } + if found.Name != "parent_strings" { + t.Errorf("expected parent template to match, got %s", found.Name) + } +} + +func TestCollectTemplatesAlongPath(t *testing.T) { + dm := NewDocumentMapping() + dm.AddDynamicTemplate(NewDynamicTemplate("root")) + + level1 := NewDocumentMapping() + level1.AddDynamicTemplate(NewDynamicTemplate("level1")) + dm.AddSubDocumentMapping("level1", level1) + + level2 := NewDocumentMapping() + level2.AddDynamicTemplate(NewDynamicTemplate("level2")) + level1.AddSubDocumentMapping("level2", level2) + + // Collect templates for a path at level 3 + templates := dm.collectTemplatesAlongPath([]string{"level1", "level2", "field"}) + + // Should have templates from root and level1 (not level2 since field is at level2) + if len(templates) != 2 { + t.Fatalf("expected 2 templates, got %d", len(templates)) + } + + names := make([]string, len(templates)) + for i, tmpl := range templates { + names[i] = tmpl.Name + } + + // Check that root template is first (inheritance order) + if templates[0].Name != "root" { + t.Errorf("expected first template to be 'root', got %s", templates[0].Name) + } + if templates[1].Name != "level1" { + t.Errorf("expected second template to be 'level1', got %s", templates[1].Name) + } +} + +func TestDynamicTemplateWithDocumentMapping(t *testing.T) { + mapping := NewIndexMapping() + + // Add a dynamic template that matches all string fields ending in _keyword + // and uses the keyword analyzer + keywordTemplate := NewDynamicTemplate("keyword_strings"). + MatchField("*_keyword"). + MatchType("string"). + WithMapping(&FieldMapping{ + Type: "text", + Analyzer: "keyword", + Store: true, + Index: true, + }) + + mapping.DefaultMapping.AddDynamicTemplate(keywordTemplate) + + // Create a document with a field that matches the template + data := map[string]interface{}{ + "name": "test", + "status_keyword": "active", + } + + doc := document.NewDocument("1") + err := mapping.MapDocument(doc, data) + if err != nil { + t.Fatalf("failed to map document: %v", err) + } + + // Verify the fields were created + foundName := false + foundStatus := false + for _, field := range doc.Fields { + switch field.Name() { + case "name": + foundName = true + case "status_keyword": + foundStatus = true + } + } + + if !foundName { + t.Error("expected to find 'name' field") + } + if !foundStatus { + t.Error("expected to find 'status_keyword' field") + } +} + +func TestDynamicTemplateWithNumbers(t *testing.T) { + mapping := NewIndexMapping() + + // Add a template that matches number fields ending in _count + // and disables doc values + countTemplate := NewDynamicTemplate("count_fields"). + MatchField("*_count"). + MatchType("number"). + WithMapping(&FieldMapping{ + Type: "number", + Store: true, + Index: true, + DocValues: false, + }) + + mapping.DefaultMapping.AddDynamicTemplate(countTemplate) + + data := map[string]interface{}{ + "item_count": 42, + "total": 100, + } + + doc := document.NewDocument("1") + err := mapping.MapDocument(doc, data) + if err != nil { + t.Fatalf("failed to map document: %v", err) + } + + foundItemCount := false + foundTotal := false + for _, field := range doc.Fields { + switch field.Name() { + case "item_count": + foundItemCount = true + case "total": + foundTotal = true + } + } + + if !foundItemCount { + t.Error("expected to find 'item_count' field") + } + if !foundTotal { + t.Error("expected to find 'total' field") + } +} + +func TestDynamicTemplateWithBooleans(t *testing.T) { + mapping := NewIndexMapping() + + // Add a template that matches boolean fields ending in _flag + flagTemplate := NewDynamicTemplate("flag_fields"). + MatchField("*_flag"). + MatchType("boolean"). + WithMapping(&FieldMapping{ + Type: "boolean", + Store: true, + Index: true, + }) + + mapping.DefaultMapping.AddDynamicTemplate(flagTemplate) + + data := map[string]interface{}{ + "active_flag": true, + "enabled": false, + } + + doc := document.NewDocument("1") + err := mapping.MapDocument(doc, data) + if err != nil { + t.Fatalf("failed to map document: %v", err) + } + + foundActiveFlag := false + foundEnabled := false + for _, field := range doc.Fields { + switch field.Name() { + case "active_flag": + foundActiveFlag = true + case "enabled": + foundEnabled = true + } + } + + if !foundActiveFlag { + t.Error("expected to find 'active_flag' field") + } + if !foundEnabled { + t.Error("expected to find 'enabled' field") + } +} + +func TestDynamicTemplateWithPathMatch(t *testing.T) { + mapping := NewIndexMapping() + + // Add a template that matches all fields under metadata.** + metadataTemplate := NewDynamicTemplate("metadata_fields"). + MatchPath("metadata.**"). + MatchType("string"). + WithMapping(&FieldMapping{ + Type: "text", + Analyzer: "keyword", + Store: true, + Index: true, + }) + + mapping.DefaultMapping.AddDynamicTemplate(metadataTemplate) + + data := map[string]interface{}{ + "title": "Test Document", + "metadata": map[string]interface{}{ + "author": "John Doe", + "tags": map[string]interface{}{ + "primary": "test", + }, + }, + } + + doc := document.NewDocument("1") + err := mapping.MapDocument(doc, data) + if err != nil { + t.Fatalf("failed to map document: %v", err) + } + + foundTitle := false + foundAuthor := false + foundPrimary := false + for _, field := range doc.Fields { + switch field.Name() { + case "title": + foundTitle = true + case "metadata.author": + foundAuthor = true + case "metadata.tags.primary": + foundPrimary = true + } + } + + if !foundTitle { + t.Error("expected to find 'title' field") + } + if !foundAuthor { + t.Error("expected to find 'metadata.author' field") + } + if !foundPrimary { + t.Error("expected to find 'metadata.tags.primary' field") + } +} + +func TestDynamicTemplateWithDateTime(t *testing.T) { + mapping := NewIndexMapping() + + // Add a template that matches datetime fields ending in _at + dateTemplate := NewDynamicTemplate("datetime_fields"). + MatchField("*_at"). + MatchType("date"). + WithMapping(&FieldMapping{ + Type: "datetime", + Store: true, + Index: true, + }) + + mapping.DefaultMapping.AddDynamicTemplate(dateTemplate) + + now := time.Now() + data := map[string]interface{}{ + "created_at": now, + "timestamp": now, + } + + doc := document.NewDocument("1") + err := mapping.MapDocument(doc, data) + if err != nil { + t.Fatalf("failed to map document: %v", err) + } + + foundCreatedAt := false + foundTimestamp := false + for _, field := range doc.Fields { + switch field.Name() { + case "created_at": + foundCreatedAt = true + case "timestamp": + foundTimestamp = true + } + } + + if !foundCreatedAt { + t.Error("expected to find 'created_at' field") + } + if !foundTimestamp { + t.Error("expected to find 'timestamp' field") + } +} + +func TestDynamicTemplateOrder(t *testing.T) { + // Test that templates are evaluated in order and first match wins + mapping := NewIndexMapping() + + // First template - more specific + specificTemplate := NewDynamicTemplate("specific"). + MatchField("special_*"). + WithMapping(&FieldMapping{ + Type: "text", + Analyzer: "keyword", + Store: true, + Index: true, + }) + + // Second template - catches all strings + catchAllTemplate := NewDynamicTemplate("catch_all"). + MatchType("string"). + WithMapping(&FieldMapping{ + Type: "text", + Analyzer: "standard", + Store: true, + Index: true, + }) + + mapping.DefaultMapping.AddDynamicTemplate(specificTemplate) + mapping.DefaultMapping.AddDynamicTemplate(catchAllTemplate) + + // Test that special_field matches the first template + found := mapping.DefaultMapping.findMatchingTemplate("special_field", "special_field", "string", nil) + if found == nil || found.Name != "specific" { + t.Error("expected 'specific' template to match first") + } + + // Test that other_field matches the second template + found = mapping.DefaultMapping.findMatchingTemplate("other_field", "other_field", "string", nil) + if found == nil || found.Name != "catch_all" { + t.Error("expected 'catch_all' template to match") + } +} + +func TestDynamicTemplateInheritanceOverride(t *testing.T) { + mapping := NewIndexMapping() + + // Parent template for all strings + parentTemplate := NewDynamicTemplate("parent_strings"). + MatchType("string"). + WithMapping(&FieldMapping{ + Type: "text", + Analyzer: "standard", + Store: true, + Index: true, + }) + mapping.DefaultMapping.AddDynamicTemplate(parentTemplate) + + // Create a child mapping that overrides for *_keyword fields + childMapping := NewDocumentMapping() + childTemplate := NewDynamicTemplate("child_keyword"). + MatchField("*_keyword"). + WithMapping(&FieldMapping{ + Type: "text", + Analyzer: "keyword", + Store: true, + Index: true, + }) + childMapping.AddDynamicTemplate(childTemplate) + mapping.DefaultMapping.AddSubDocumentMapping("nested", childMapping) + + // Collect parent templates + parentTemplates := mapping.DefaultMapping.DynamicTemplates + + // Test that child template takes precedence for *_keyword + found := childMapping.findMatchingTemplate("status_keyword", "nested.status_keyword", "string", parentTemplates) + if found == nil || found.Name != "child_keyword" { + t.Error("expected 'child_keyword' template to match") + } + + // Test that parent template is used for other strings + found = childMapping.findMatchingTemplate("title", "nested.title", "string", parentTemplates) + if found == nil || found.Name != "parent_strings" { + t.Error("expected 'parent_strings' template to match") + } +} diff --git a/mapping/field.go b/mapping/field.go index 0b6074910..c66301a79 100644 --- a/mapping/field.go +++ b/mapping/field.go @@ -175,6 +175,76 @@ func newBooleanFieldMappingDynamic(im *IndexMappingImpl) *FieldMapping { return rv } +// applyDynamicDefaults creates a copy of the field mapping and applies +// dynamic defaults from the index mapping for fields that aren't explicitly set. +// This is used when applying dynamic templates to ensure global dynamic settings +// are respected unless the template explicitly overrides them. +func applyDynamicDefaults(fm *FieldMapping, im *IndexMappingImpl) *FieldMapping { + // Create a copy of the field mapping + rv := &FieldMapping{ + Name: fm.Name, + Type: fm.Type, + Analyzer: fm.Analyzer, + Store: fm.Store, + Index: fm.Index, + IncludeTermVectors: fm.IncludeTermVectors, + IncludeInAll: fm.IncludeInAll, + DateFormat: fm.DateFormat, + DocValues: fm.DocValues, + SkipFreqNorm: fm.SkipFreqNorm, + Dims: fm.Dims, + Similarity: fm.Similarity, + VectorIndexOptimizedFor: fm.VectorIndexOptimizedFor, + SynonymSource: fm.SynonymSource, + } + + // If the template didn't explicitly set these values, use dynamic defaults + // We check the zero values to see if they were explicitly set + // Note: This is a simplification; ideally we'd track which fields were set + if !fm.Store && !fm.Index && !fm.DocValues { + // No explicit indexing options set, apply defaults based on type + switch fm.Type { + case "text": + rv.Store = im.StoreDynamic + rv.Index = im.IndexDynamic + rv.DocValues = im.DocValuesDynamic + rv.IncludeTermVectors = true + rv.IncludeInAll = true + case "number": + rv.Store = im.StoreDynamic + rv.Index = im.IndexDynamic + rv.DocValues = im.DocValuesDynamic + rv.IncludeInAll = true + case "datetime": + rv.Store = im.StoreDynamic + rv.Index = im.IndexDynamic + rv.DocValues = im.DocValuesDynamic + rv.IncludeInAll = true + case "boolean": + rv.Store = im.StoreDynamic + rv.Index = im.IndexDynamic + rv.DocValues = im.DocValuesDynamic + rv.IncludeInAll = true + case "geopoint": + rv.Store = im.StoreDynamic + rv.Index = im.IndexDynamic + rv.DocValues = im.DocValuesDynamic + rv.IncludeInAll = true + case "IP": + rv.Store = im.StoreDynamic + rv.Index = im.IndexDynamic + rv.IncludeInAll = true + default: + // For unknown types, apply basic defaults + rv.Store = im.StoreDynamic + rv.Index = im.IndexDynamic + rv.DocValues = im.DocValuesDynamic + } + } + + return rv +} + // NewGeoPointFieldMapping returns a default field mapping for geo points func NewGeoPointFieldMapping() *FieldMapping { return &FieldMapping{