diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 594b27f029901..e26c969f7d495 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -209,7 +209,8 @@ public static final class Builder extends FieldMapper.DimensionBuilder { private final IndexAnalyzers indexAnalyzers; private final ScriptCompiler scriptCompiler; private final IndexVersion indexCreatedVersion; - private final boolean useDocValuesSkipper; + private final boolean enableDocValuesSkipper; + private final boolean forceDocValuesSkipper; private final SourceKeepMode indexSourceKeepMode; public Builder(final String name, final MappingParserContext mappingParserContext) { @@ -222,6 +223,7 @@ public Builder(final String name, final MappingParserContext mappingParserContex mappingParserContext.getIndexSettings().getMode(), mappingParserContext.getIndexSettings().getIndexSortConfig(), USE_DOC_VALUES_SKIPPER.get(mappingParserContext.getSettings()), + false, mappingParserContext.getIndexSettings().sourceKeepMode() ); } @@ -243,6 +245,7 @@ public Builder(final String name, final MappingParserContext mappingParserContex IndexMode.STANDARD, null, false, + false, sourceKeepMode ); } @@ -255,7 +258,8 @@ private Builder( IndexVersion indexCreatedVersion, IndexMode indexMode, IndexSortConfig indexSortConfig, - boolean useDocValuesSkipper, + boolean enableDocValuesSkipper, + boolean forceDocValuesSkipper, SourceKeepMode indexSourceKeepMode ) { super(name); @@ -293,7 +297,8 @@ private Builder( }); this.indexSortConfig = indexSortConfig; this.indexMode = indexMode; - this.useDocValuesSkipper = useDocValuesSkipper; + this.enableDocValuesSkipper = enableDocValuesSkipper; + this.forceDocValuesSkipper = forceDocValuesSkipper; this.indexSourceKeepMode = indexSourceKeepMode; } @@ -301,6 +306,27 @@ public Builder(String name, IndexVersion indexCreatedVersion) { this(name, null, ScriptCompiler.NONE, Integer.MAX_VALUE, indexCreatedVersion, SourceKeepMode.NONE); } + public static Builder buildWithDocValuesSkipper( + String name, + IndexMode indexMode, + IndexVersion indexCreatedVersion, + boolean enableDocValuesSkipper + ) { + return new Builder( + name, + null, + ScriptCompiler.NONE, + Integer.MAX_VALUE, + indexCreatedVersion, + indexMode, + // Sort config is used to decide if DocValueSkippers can be used. Since skippers are forced, a sort config is not needed. + null, + enableDocValuesSkipper, + true, + SourceKeepMode.NONE + ); + } + public Builder ignoreAbove(int ignoreAbove) { this.ignoreAbove.setValue(ignoreAbove); return this; @@ -422,7 +448,9 @@ private KeywordFieldType buildFieldType(MapperBuilderContext context, FieldType @Override public KeywordFieldMapper build(MapperBuilderContext context) { FieldType fieldtype = resolveFieldType( - useDocValuesSkipper, + enableDocValuesSkipper, + forceDocValuesSkipper, + hasDocValues, indexCreatedVersion, indexSortConfig, indexMode, @@ -460,24 +488,30 @@ public KeywordFieldMapper build(MapperBuilderContext context) { buildFieldType(context, fieldtype), builderParams(this, context), context.isSourceSynthetic(), - useDocValuesSkipper, this, offsetsFieldName, indexSourceKeepMode ); } - private FieldType resolveFieldType( - final boolean useDocValuesSkipper, + private static FieldType resolveFieldType( + final boolean enableDocValuesSkipper, + final boolean forceDocValuesSkipper, + final Parameter hasDocValues, final IndexVersion indexCreatedVersion, final IndexSortConfig indexSortConfig, final IndexMode indexMode, final String fullFieldName ) { - if (useDocValuesSkipper - && indexCreatedVersion.onOrAfter(IndexVersions.HOSTNAME_DOC_VALUES_SPARSE_INDEX) - && shouldUseDocValuesSkipper(hasDocValues.getValue(), indexSortConfig, indexMode, fullFieldName)) { - return new FieldType(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES); + if (enableDocValuesSkipper) { + if (forceDocValuesSkipper) { + assert hasDocValues.getValue(); + return new FieldType(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES); + } + if (indexCreatedVersion.onOrAfter(IndexVersions.HOSTNAME_DOC_VALUES_SPARSE_INDEX) + && shouldUseDocValuesSkipper(hasDocValues.getValue(), indexSortConfig, indexMode, fullFieldName)) { + return new FieldType(Defaults.FIELD_TYPE_WITH_SKIP_DOC_VALUES); + } } return new FieldType(Defaults.FIELD_TYPE); } @@ -1088,7 +1122,8 @@ public String originalName() { private final int ignoreAboveDefault; private final IndexMode indexMode; private final IndexSortConfig indexSortConfig; - private final boolean useDocValuesSkipper; + private final boolean enableDocValuesSkipper; + private final boolean forceDocValuesSkipper; private final String offsetsFieldName; private final SourceKeepMode indexSourceKeepMode; private final String originalName; @@ -1099,7 +1134,6 @@ private KeywordFieldMapper( KeywordFieldType mappedFieldType, BuilderParams builderParams, boolean isSyntheticSource, - boolean useDocValuesSkipper, Builder builder, String offsetsFieldName, SourceKeepMode indexSourceKeepMode @@ -1120,7 +1154,8 @@ private KeywordFieldMapper( this.ignoreAboveDefault = builder.ignoreAboveDefault; this.indexMode = builder.indexMode; this.indexSortConfig = builder.indexSortConfig; - this.useDocValuesSkipper = useDocValuesSkipper; + this.enableDocValuesSkipper = builder.enableDocValuesSkipper; + this.forceDocValuesSkipper = builder.forceDocValuesSkipper; this.offsetsFieldName = offsetsFieldName; this.indexSourceKeepMode = indexSourceKeepMode; this.originalName = mappedFieldType.originalName(); @@ -1219,7 +1254,7 @@ private boolean indexValue(DocumentParserContext context, XContentString value) throw new IllegalArgumentException(msg); } - Field field = new KeywordField(fieldType().name(), binaryValue, fieldType); + Field field = buildKeywordField(binaryValue); context.doc().add(field); if (fieldType().hasDocValues() == false && fieldType.omitNorms()) { @@ -1276,11 +1311,16 @@ public FieldMapper.Builder getMergeBuilder() { indexCreatedVersion, indexMode, indexSortConfig, - useDocValuesSkipper, + enableDocValuesSkipper, + forceDocValuesSkipper, indexSourceKeepMode ).dimension(fieldType().isDimension()).init(this); } + public Field buildKeywordField(BytesRef binaryValue) { + return new KeywordField(fieldType().name(), binaryValue, fieldType); + } + @Override public void doValidate(MappingLookup lookup) { if (fieldType().isDimension() && null != lookup.nestedLookup().getNestedParent(fullPath())) { diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapper.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapper.java index 55f5616f4ac74..771e06da4a97b 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapper.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapper.java @@ -13,19 +13,28 @@ import org.apache.lucene.index.IndexOptions; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.util.FeatureFlag; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.mapper.CompositeSyntheticFieldLoader; import org.elasticsearch.index.mapper.DocumentParserContext; import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.KeywordFieldMapper; +import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperBuilderContext; +import org.elasticsearch.index.mapper.MappingParserContext; import org.elasticsearch.index.mapper.TextParams; import org.elasticsearch.index.mapper.TextSearchInfo; import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; import java.util.Map; +import static org.elasticsearch.index.IndexSettings.USE_DOC_VALUES_SKIPPER; + /** * A {@link FieldMapper} that assigns every document the same value. */ @@ -50,20 +59,38 @@ public static class Defaults { public static class Builder extends FieldMapper.Builder { private final IndexVersion indexCreatedVersion; - + private final IndexSettings indexSettings; private final Parameter> meta = Parameter.metaParam(); - private final TextParams.Analyzers analyzers; + private final boolean enableDocValuesSkipper; + + public Builder(String name, MappingParserContext context) { + this( + name, + context.indexVersionCreated(), + context.getIndexSettings(), + context.getIndexAnalyzers(), + USE_DOC_VALUES_SKIPPER.get(context.getSettings()) + ); + } - public Builder(String name, IndexVersion indexCreatedVersion, IndexAnalyzers indexAnalyzers) { + public Builder( + String name, + IndexVersion indexCreatedVersion, + IndexSettings indexSettings, + IndexAnalyzers indexAnalyzers, + boolean enableDocValuesSkipper + ) { super(name); this.indexCreatedVersion = indexCreatedVersion; + this.indexSettings = indexSettings; this.analyzers = new TextParams.Analyzers( indexAnalyzers, m -> ((PatternedTextFieldMapper) m).indexAnalyzer, m -> ((PatternedTextFieldMapper) m).positionIncrementGap, indexCreatedVersion ); + this.enableDocValuesSkipper = enableDocValuesSkipper; } @Override @@ -87,23 +114,35 @@ private PatternedTextFieldType buildFieldType(MapperBuilderContext context) { @Override public PatternedTextFieldMapper build(MapperBuilderContext context) { - return new PatternedTextFieldMapper(leafName(), buildFieldType(context), builderParams(this, context), this); + PatternedTextFieldType patternedTextFieldType = buildFieldType(context); + BuilderParams builderParams = builderParams(this, context); + var templateIdMapper = KeywordFieldMapper.Builder.buildWithDocValuesSkipper( + patternedTextFieldType.templateIdFieldName(), + indexSettings.getMode(), + indexCreatedVersion, + enableDocValuesSkipper + ).build(context); + return new PatternedTextFieldMapper(leafName(), patternedTextFieldType, builderParams, this, templateIdMapper); } } - public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.indexVersionCreated(), c.getIndexAnalyzers())); + public static final TypeParser PARSER = new TypeParser(Builder::new); private final IndexVersion indexCreatedVersion; private final IndexAnalyzers indexAnalyzers; + private final IndexSettings indexSettings; private final NamedAnalyzer indexAnalyzer; + private final boolean enableDocValuesSkipper; private final int positionIncrementGap; private final FieldType fieldType; + private final KeywordFieldMapper templateIdMapper; private PatternedTextFieldMapper( String simpleName, PatternedTextFieldType mappedFieldPatternedTextFieldType, BuilderParams builderParams, - Builder builder + Builder builder, + KeywordFieldMapper templateIdMapper ) { super(simpleName, mappedFieldPatternedTextFieldType, builderParams); assert mappedFieldPatternedTextFieldType.getTextSearchInfo().isTokenized(); @@ -112,7 +151,10 @@ private PatternedTextFieldMapper( this.indexCreatedVersion = builder.indexCreatedVersion; this.indexAnalyzers = builder.analyzers.indexAnalyzers; this.indexAnalyzer = builder.analyzers.getIndexAnalyzer(); + this.indexSettings = builder.indexSettings; + this.enableDocValuesSkipper = builder.enableDocValuesSkipper; this.positionIncrementGap = builder.analyzers.positionIncrementGap.getValue(); + this.templateIdMapper = templateIdMapper; } @Override @@ -122,7 +164,18 @@ public Map indexAnalyzers() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), indexCreatedVersion, indexAnalyzers).init(this); + return new Builder(leafName(), indexCreatedVersion, indexSettings, indexAnalyzers, enableDocValuesSkipper).init(this); + } + + @Override + public Iterator iterator() { + List mappers = new ArrayList<>(); + Iterator m = super.iterator(); + while (m.hasNext()) { + mappers.add(m.next()); + } + mappers.add(templateIdMapper); + return mappers.iterator(); } @Override @@ -146,6 +199,9 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio // Add template doc_values context.doc().add(new SortedSetDocValuesField(fieldType().templateFieldName(), new BytesRef(parts.template()))); + // Add template_id doc_values + context.doc().add(templateIdMapper.buildKeywordField(new BytesRef(parts.templateId()))); + // Add args doc_values if (parts.args().isEmpty() == false) { String remainingArgs = PatternedTextValueProcessor.encodeRemainingArgs(parts); diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java index 4c712d10e0aa4..e23e1428fbe24 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldType.java @@ -55,6 +55,7 @@ public class PatternedTextFieldType extends StringFieldType { private static final String TEMPLATE_SUFFIX = ".template"; + private static final String TEMPLATE_ID_SUFFIX = ".template_id"; private static final String ARGS_SUFFIX = ".args"; public static final String CONTENT_TYPE = "patterned_text"; @@ -145,7 +146,7 @@ public Query fuzzyQuery( @Override public Query existsQuery(SearchExecutionContext context) { - return new FieldExistsQuery(templateFieldName()); + return new FieldExistsQuery(templateIdFieldName()); } @Override @@ -263,6 +264,10 @@ String templateFieldName() { return name() + TEMPLATE_SUFFIX; } + String templateIdFieldName() { + return name() + TEMPLATE_ID_SUFFIX; + } + String argsFieldName() { return name() + ARGS_SUFFIX; } diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextValueProcessor.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextValueProcessor.java index c4551777c319f..70fdb97cfd6c3 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextValueProcessor.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextValueProcessor.java @@ -7,6 +7,11 @@ package org.elasticsearch.xpack.logsdb.patternedtext; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.hash.MurmurHash3; +import org.elasticsearch.common.util.ByteUtils; + +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -16,7 +21,20 @@ public class PatternedTextValueProcessor { private static final String DELIMITER = "[\\s\\[\\]]"; private static final String SPACE = " "; - record Parts(String template, List args) {} + record Parts(String template, String templateId, List args) { + Parts(String template, List args) { + this(template, PatternedTextValueProcessor.templateId(template), args); + } + } + + static String templateId(String template) { + byte[] bytes = template.getBytes(StandardCharsets.UTF_8); + MurmurHash3.Hash128 hash = new MurmurHash3.Hash128(); + MurmurHash3.hash128(bytes, 0, bytes.length, 0, hash); + byte[] hashBytes = new byte[8]; + ByteUtils.writeLongLE(hash.h1, hashBytes, 0); + return Strings.BASE_64_NO_PADDING_URL_ENCODER.encodeToString(hashBytes); + } static Parts split(String text) { StringBuilder template = new StringBuilder(); diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapperTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapperTests.java index 9b572e171bd96..a6c2f08e059ab 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapperTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextFieldMapperTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.Tuple; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.DocumentMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; @@ -51,7 +52,6 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; -import static org.hamcrest.Matchers.startsWith; public class PatternedTextFieldMapperTests extends MapperTestCase { @@ -69,7 +69,7 @@ protected Object getSampleValueForDocument() { protected void assertExistsQuery(MappedFieldType fieldType, Query query, LuceneDocument fields) { assertThat(query, instanceOf(FieldExistsQuery.class)); FieldExistsQuery fieldExistsQuery = (FieldExistsQuery) query; - assertThat(fieldExistsQuery.getField(), startsWith("field")); + assertThat(fieldExistsQuery.getField(), equalTo("field.template_id")); assertNoFieldNamesField(fields); } @@ -131,23 +131,44 @@ protected void minimalStoreMapping(XContentBuilder b) throws IOException { } public void testDefaults() throws IOException { - DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); + boolean enabledDocValuesSkipper = randomBoolean(); + var indexSettings = getIndexSettingsBuilder().put(IndexSettings.USE_DOC_VALUES_SKIPPER.getKey(), enabledDocValuesSkipper).build(); + + DocumentMapper mapper = createMapperService(indexSettings, fieldMapping(this::minimalMapping)).documentMapper(); assertEquals(Strings.toString(fieldMapping(this::minimalMapping)), mapper.mappingSource().toString()); ParsedDocument doc = mapper.parse(source(b -> b.field("field", "1234"))); - List fields = doc.rootDoc().getFields("field"); - assertEquals(1, fields.size()); - assertEquals("1234", fields.get(0).stringValue()); - IndexableFieldType fieldType = fields.get(0).fieldType(); - assertThat(fieldType.omitNorms(), equalTo(true)); - assertTrue(fieldType.tokenized()); - assertFalse(fieldType.stored()); - assertThat(fieldType.indexOptions(), equalTo(IndexOptions.DOCS)); - assertThat(fieldType.storeTermVectors(), equalTo(false)); - assertThat(fieldType.storeTermVectorOffsets(), equalTo(false)); - assertThat(fieldType.storeTermVectorPositions(), equalTo(false)); - assertThat(fieldType.storeTermVectorPayloads(), equalTo(false)); - assertEquals(DocValuesType.NONE, fieldType.docValuesType()); + { + List fields = doc.rootDoc().getFields("field"); + assertEquals(1, fields.size()); + assertEquals("1234", fields.get(0).stringValue()); + IndexableFieldType fieldType = fields.get(0).fieldType(); + assertThat(fieldType.omitNorms(), equalTo(true)); + assertTrue(fieldType.tokenized()); + assertFalse(fieldType.stored()); + assertThat(fieldType.indexOptions(), equalTo(IndexOptions.DOCS)); + assertThat(fieldType.storeTermVectors(), equalTo(false)); + assertThat(fieldType.storeTermVectorOffsets(), equalTo(false)); + assertThat(fieldType.storeTermVectorPositions(), equalTo(false)); + assertThat(fieldType.storeTermVectorPayloads(), equalTo(false)); + assertEquals(DocValuesType.NONE, fieldType.docValuesType()); + } + + { + List fields = doc.rootDoc().getFields("field.template_id"); + assertEquals(1, fields.size()); + assertEquals("D3OycqSEnDM", fields.get(0).binaryValue().utf8ToString()); + IndexableFieldType fieldType = fields.get(0).fieldType(); + assertThat(fieldType.omitNorms(), equalTo(true)); + assertFalse(fieldType.tokenized()); + assertFalse(fieldType.stored()); + assertThat(fieldType.indexOptions(), equalTo(enabledDocValuesSkipper ? IndexOptions.NONE : IndexOptions.DOCS)); + assertThat(fieldType.storeTermVectors(), equalTo(false)); + assertThat(fieldType.storeTermVectorOffsets(), equalTo(false)); + assertThat(fieldType.storeTermVectorPositions(), equalTo(false)); + assertThat(fieldType.storeTermVectorPayloads(), equalTo(false)); + assertEquals(DocValuesType.SORTED_SET, fieldType.docValuesType()); + } } public void testNullConfigValuesFail() throws MapperParsingException { diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextValueProcessorTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextValueProcessorTests.java index 58266b3dae1ea..fe496fdaeb558 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextValueProcessorTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/patternedtext/PatternedTextValueProcessorTests.java @@ -10,6 +10,10 @@ import org.elasticsearch.test.ESTestCase; import org.hamcrest.Matchers; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + public class PatternedTextValueProcessorTests extends ESTestCase { public void testEmpty() { @@ -98,4 +102,43 @@ public void testWithTimestamp1() { assertThat(parts.args(), Matchers.contains("2020-08-18T00:58:56", "123", "cheddar1")); assertEquals(text, PatternedTextValueProcessor.merge(parts)); } + + public void testTemplateIdIsExpectedShape() { + String text = "[2020-08-18T00:58:56] Found 123 errors for service [cheddar1]"; + PatternedTextValueProcessor.Parts parts = PatternedTextValueProcessor.split(text); + assertEquals("vSr1YMYPups", parts.templateId()); + } + + public void testTemplateIdHasVeryFewCollisions() { + Set templates = new HashSet<>(); + Set ids = new HashSet<>(); + + for (int i = 0; i < 1000; i++) { + var template = randomTemplate(); + var parts = new PatternedTextValueProcessor.Parts(template, List.of()); + templates.add(template); + ids.add(parts.templateId()); + } + // This can technically fail due to hash collision, but it should happen quite rarely. + assertEquals(templates.size(), ids.size()); + } + + private static String randomTemplate() { + StringBuilder sb = new StringBuilder(); + int numTokens = randomIntBetween(1, 20); + for (int i = 0; i < numTokens; i++) { + var token = randomBoolean() ? randomAlphaOfLength(between(1, 10)) : randomPlaceholder(); + sb.append(token); + sb.append(randomDelimiter()); + } + return sb.toString(); + } + + private static String randomPlaceholder() { + return randomFrom(List.of("%W", "%D", "%I", "%U", "%T")); + } + + private static String randomDelimiter() { + return randomFrom(List.of(" ", "\n", "\t", "[", "]")); + } } diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/patternedtext/30_template_id.yml b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/patternedtext/30_template_id.yml new file mode 100644 index 0000000000000..4d63c8727b10e --- /dev/null +++ b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/patternedtext/30_template_id.yml @@ -0,0 +1,196 @@ +setup: + - requires: + cluster_features: [ "mapper.patterned_text" ] + reason: "patterned_text mappings are used in this test" + + - do: + indices.create: + index: test + body: + settings: + index.mapping.use_doc_values_skipper: true + index.mode: logsdb + sort.field: [ "foo.template_id" ] + sort.order: [ desc ] + mappings: + properties: + foo: + type: patterned_text + + - do: + index: + index: test + id: "1" + body: + "@timestamp": 2025-07-17T00:00:00Z + + - do: + index: + index: test + id: "2" + body: + "foo": "Found 5 errors for service [cheddar1]" + "@timestamp": 2025-07-17T00:00:01Z + + - do: + index: + index: test + id: "3" + body: + "foo": "[2020-08-18T00:58:56] Found 123 errors for service [cheddar1]" + "@timestamp": 2025-07-17T00:00:02Z + + - do: + index: + index: test + id: "4" + body: + "foo": "Found some errors for cheddar data service" + "@timestamp": 2025-07-17T00:00:03Z + + - do: + index: + index: test + id: "5" + body: + "foo": "Found 123 errors for service [gorgonzola-24]" + "@timestamp": 2025-07-17T00:00:04Z + + - do: + indices.refresh: {} + +--- +Field caps: + + - do: + field_caps: + index: test + fields: [ foo.template_id ] + - match: { fields: {"foo.template_id": {"keyword": {"type": "keyword", "metadata_field": false, "searchable": true, "aggregatable": true}}}} + +--- +Get template_id field: + + - do: + search: + index: test + body: + docvalue_fields: [ "foo.template_id" ] + query: + ids: + values: ["2"] + + - match: { hits.total.value: 1 } + - match: { hits.hits.0.fields: {"foo.template_id": ["iJEgF75EQNk"]} } + +--- +Sort by template_id index config setting: + - do: + search: + index: test + body: + docvalue_fields: [ "foo.template_id" ] + query: { match_all: {} } + + - match: { hits.total.value: 5 } + - match: { hits.hits.0.fields: {"foo.template_id": ["vSr1YMYPups"]} } + - match: { hits.hits.0._id: "3" } + - match: { hits.hits.1.fields: {"foo.template_id": ["k-2qtjujOCw"]} } + - match: { hits.hits.1._id: "4" } + - match: { hits.hits.2.fields: {"foo.template_id": ["iJEgF75EQNk"]} } + - match: { hits.hits.2._id: "2" } + - match: { hits.hits.3.fields: {"foo.template_id": ["iJEgF75EQNk"]} } + - match: { hits.hits.3._id: "5" } + - match: { hits.hits.4.fields: null } + - match: { hits.hits.4._id: "1" } + +--- +Sort by template_id: + + - do: + search: + index: test + body: + docvalue_fields: [ "foo.template_id" ] + sort: "foo.template_id" + query: { match_all: {} } + + - match: { hits.total.value: 5 } + - match: { hits.hits.0.fields: {"foo.template_id": ["iJEgF75EQNk"]} } + - match: { hits.hits.0._id: "2" } + - match: { hits.hits.1.fields: {"foo.template_id": ["iJEgF75EQNk"]} } + - match: { hits.hits.1._id: "5" } + - match: { hits.hits.2.fields: {"foo.template_id": ["k-2qtjujOCw"]} } + - match: { hits.hits.2._id: "4" } + - match: { hits.hits.3.fields: {"foo.template_id": ["vSr1YMYPups"]} } + - match: { hits.hits.3._id: "3" } + - match: { hits.hits.4.fields: null } + - match: { hits.hits.4._id: "1" } + +--- +Exist query: + + - do: + search: + index: test + body: + query: + exists: + field: foo.template_id + + - match: { hits.total.value: 4 } + - match: { hits.hits.0._score: 1.0 } + +--- +Match query: + + - do: + search: + index: test + body: + query: + match: + foo.template_id: "iJEgF75EQNk" + + - match: { hits.total.value: 2 } + - match: { hits.hits.0._score: 1.0 } + +--- +Range query: + + - do: + search: + index: test + body: + query: + range: + foo.template_id: + gt: "j" + # one doc has a null value for template_id and two start with i < j + - match: { hits.total.value: 2 } + - match: { hits.hits.0._id: "3" } + - match: { hits.hits.1._id: "4" } + +--- +Term aggregation: + - do: + search: + index: test + body: + size: 0 + aggs: + template_id_agg: + terms: + field: "foo.template_id" + "order": { "_key": "asc" } + + - match: { hits.total.value: 5 } + - length: { aggregations.template_id_agg.buckets: 3 } + - match: { aggregations.template_id_agg.buckets.0.key: "iJEgF75EQNk" } + - match: { aggregations.template_id_agg.buckets.0.doc_count: 2 } + - match: { aggregations.template_id_agg.buckets.1.key: "k-2qtjujOCw" } + - match: { aggregations.template_id_agg.buckets.1.doc_count: 1 } + - match: { aggregations.template_id_agg.buckets.2.key: "vSr1YMYPups" } + - match: { aggregations.template_id_agg.buckets.2.doc_count: 1 } + +