From 0cfdf33c7d254d7d7f34b1dd183a6de2c308d607 Mon Sep 17 00:00:00 2001 From: Luca Cavanna Date: Wed, 17 Jun 2020 16:10:15 +0200 Subject: [PATCH 01/35] Add runtime fields plugin under x-pack --- x-pack/plugin/runtime-fields/build.gradle | 26 ++++++++++++++++ .../xpack/runtimefields/RuntimeFields.java | 30 +++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 x-pack/plugin/runtime-fields/build.gradle create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java diff --git a/x-pack/plugin/runtime-fields/build.gradle b/x-pack/plugin/runtime-fields/build.gradle new file mode 100644 index 0000000000000..4b9ae1edb7d93 --- /dev/null +++ b/x-pack/plugin/runtime-fields/build.gradle @@ -0,0 +1,26 @@ +evaluationDependsOn(xpackModule('core')) + +apply plugin: 'elasticsearch.esplugin' + +esplugin { + name 'x-pack-runtime-fields' + description 'A module which adds support for runtime fields' + classname 'org.elasticsearch.xpack.runtimefields.RuntimeFields' + extendedPlugins = ['x-pack-core'] +} +archivesBaseName = 'x-pack-runtime-fields' + +compileJava.options.compilerArgs << "-Xlint:-rawtypes" +compileTestJava.options.compilerArgs << "-Xlint:-rawtypes" + +dependencies { + compileOnly project(":server") + + compileOnly project(path: xpackModule('core'), configuration: 'default') +} + +dependencyLicenses { + ignoreSha 'x-pack-core' +} + +integTest.enabled = false diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java new file mode 100644 index 0000000000000..2f3bf46bf3c47 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.plugins.MapperPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.ScriptPlugin; +import org.elasticsearch.script.ScriptContext; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +public final class RuntimeFields extends Plugin implements MapperPlugin, ScriptPlugin { + + @Override + public Map getMappers() { + return Collections.emptyMap(); + } + + @Override + public List> getContexts() { + return Collections.emptyList(); + } +} From e47458253c0dde470365aa81151038fca09b348b Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 17 Jun 2020 15:37:31 -0400 Subject: [PATCH 02/35] Hack together some script contexts Put together some example script contexts for scripted fields. This is almost certainly wrong, but it gets us something to think about. --- .../search/lookup/DocLookup.java | 2 +- x-pack/plugin/runtime-fields/build.gradle | 2 +- .../AbstractScriptFieldsScript.java | 63 ++++++++++ .../LongArrayScriptFieldScript.java | 34 +++++ .../runtimefields/LongScriptFieldScript.java | 34 +++++ .../xpack/runtimefields/RuntimeFields.java | 6 +- .../StringScriptFieldsScript.java | 34 +++++ .../LongArrayScriptFieldScriptTests.java | 87 +++++++++++++ .../LongScriptFieldScriptTests.java | 81 ++++++++++++ .../ScriptFieldScriptTestCase.java | 117 ++++++++++++++++++ .../StringScriptFieldsScriptTests.java | 80 ++++++++++++ 11 files changed, 537 insertions(+), 3 deletions(-) create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldsScript.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScript.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScript.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScriptTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScriptTests.java diff --git a/server/src/main/java/org/elasticsearch/search/lookup/DocLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/DocLookup.java index 0022cdfdc9d99..1f0e53fca14c9 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/DocLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/DocLookup.java @@ -30,7 +30,7 @@ public class DocLookup { private final MapperService mapperService; private final Function> fieldDataLookup; - DocLookup(MapperService mapperService, Function> fieldDataLookup) { + public DocLookup(MapperService mapperService, Function> fieldDataLookup) { this.mapperService = mapperService; this.fieldDataLookup = fieldDataLookup; } diff --git a/x-pack/plugin/runtime-fields/build.gradle b/x-pack/plugin/runtime-fields/build.gradle index 4b9ae1edb7d93..b1fbb6b839f63 100644 --- a/x-pack/plugin/runtime-fields/build.gradle +++ b/x-pack/plugin/runtime-fields/build.gradle @@ -15,7 +15,7 @@ compileTestJava.options.compilerArgs << "-Xlint:-rawtypes" dependencies { compileOnly project(":server") - + compileOnly project(':modules:lang-painless') // TODO we don't want to depend on painless if we're not extending it. But it is convenient for testing now. compileOnly project(path: xpackModule('core'), configuration: 'default') } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldsScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldsScript.java new file mode 100644 index 0000000000000..95c419b6ad558 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldsScript.java @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.script.AggregationScript; +import org.elasticsearch.search.lookup.DocLookup; +import org.elasticsearch.search.lookup.LeafDocLookup; +import org.elasticsearch.search.lookup.SourceLookup; + +import java.util.Map; + +/** + * Abstract base for scripts to execute to build scripted fields. Inspired by + * {@link AggregationScript} but hopefully with less historical baggage. + */ +public abstract class AbstractScriptFieldsScript { + private final Map params; + private final LeafReaderContext ctx; + private final SourceLookup source; + private final LeafDocLookup fieldData; + + public AbstractScriptFieldsScript(Map params, SourceLookup source, DocLookup fieldData, LeafReaderContext ctx) { + this.params = params; + this.source = source; + this.fieldData = fieldData.getLeafDocLookup(ctx); + this.ctx = ctx; + } + + /** + * Set the document to run the script against. + */ + public void setDocument(int docId) { + source.setSegmentAndDocument(ctx, docId); + fieldData.setDocument(docId); + } + + /** + * Expose the {@code params} of the script to the script itself. + */ + public Map getParams() { + return params; + } + + /** + * Expose the {@code _source} to the script. + */ + public Map getSource() { + return source; + } + + /** + * Expose field data to the script as {@code doc}. + */ + public Map> getDoc() { + return fieldData; + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScript.java new file mode 100644 index 0000000000000..e9676f580b45f --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScript.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptFactory; +import org.elasticsearch.search.lookup.DocLookup; +import org.elasticsearch.search.lookup.SourceLookup; + +import java.io.IOException; +import java.util.Map; + +public abstract class LongArrayScriptFieldScript extends AbstractScriptFieldsScript { + public static final ScriptContext CONTEXT = new ScriptContext<>("long_array_script_field", Factory.class); + public static final String[] PARAMETERS = {}; + + public interface Factory extends ScriptFactory { + LeafFactory newFactory(Map params, SourceLookup source, DocLookup fieldData); + } + public static interface LeafFactory { + LongArrayScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException; + } + + public LongArrayScriptFieldScript(Map params, SourceLookup source, DocLookup fieldData, LeafReaderContext ctx) { + super(params, source, fieldData, ctx); + } + + public abstract long[] execute(); +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java new file mode 100644 index 0000000000000..4e1dce5c9facc --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptFactory; +import org.elasticsearch.search.lookup.DocLookup; +import org.elasticsearch.search.lookup.SourceLookup; + +import java.io.IOException; +import java.util.Map; + +public abstract class LongScriptFieldScript extends AbstractScriptFieldsScript { + public static final ScriptContext CONTEXT = new ScriptContext<>("long_script_field", Factory.class); + public static final String[] PARAMETERS = {}; + + public interface Factory extends ScriptFactory { + LeafFactory newFactory(Map params, SourceLookup source, DocLookup fieldData); + } + public static interface LeafFactory { + LongScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException; + } + + public LongScriptFieldScript(Map params, SourceLookup source, DocLookup fieldData, LeafReaderContext ctx) { + super(params, source, fieldData, ctx); + } + + public abstract long execute(); +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java index 2f3bf46bf3c47..0a91bd6db2021 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java @@ -25,6 +25,10 @@ public Map getMappers() { @Override public List> getContexts() { - return Collections.emptyList(); + return List.of( + LongArrayScriptFieldScript.CONTEXT, + LongScriptFieldScript.CONTEXT, + StringScriptFieldsScript.CONTEXT + ); } } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScript.java new file mode 100644 index 0000000000000..b8f3809de3af3 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScript.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptFactory; +import org.elasticsearch.search.lookup.DocLookup; +import org.elasticsearch.search.lookup.SourceLookup; + +import java.io.IOException; +import java.util.Map; + +public abstract class StringScriptFieldsScript extends AbstractScriptFieldsScript { + public static final ScriptContext CONTEXT = new ScriptContext<>("string_script_field", Factory.class); + public static final String[] PARAMETERS = {}; + + public interface Factory extends ScriptFactory { + LeafFactory newFactory(Map params, SourceLookup source, DocLookup fieldData); + } + public static interface LeafFactory { + StringScriptFieldsScript newInstance(LeafReaderContext ctx) throws IOException; + } + + public StringScriptFieldsScript(Map params, SourceLookup source, DocLookup fieldData, LeafReaderContext ctx) { + super(params, source, fieldData, ctx); + } + + public abstract String execute(); +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScriptTests.java new file mode 100644 index 0000000000000..6a3114bd15698 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScriptTests.java @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType; +import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.search.lookup.DocLookup; +import org.elasticsearch.search.lookup.SourceLookup; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; + +public class LongArrayScriptFieldScriptTests extends ScriptFieldScriptTestCase< + LongArrayScriptFieldScript, + LongArrayScriptFieldScript.Factory, + LongArrayScriptFieldScript.LeafFactory, + Long> { + + public void testConstant() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); + }; + assertThat(execute(indexBuilder, "new long[] {10, 100}"), equalTo(List.of(10L, 100L, 10L, 100L))); + } + + public void testSource() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 10, \"bar\": 20}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 100, \"bar\": 200}")))); + }; + assertThat(execute(indexBuilder, "new long[] {source['foo'], source['bar']}"), equalTo(List.of(10L, 20L, 100L, 200L))); + } + + public void testDocValues() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", 10), new SortedNumericDocValuesField("foo", 20))); + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", 100), new SortedNumericDocValuesField("foo", 200))); + }; + assertThat( + execute(indexBuilder, "def foo = doc['foo']; new long[] {foo[0], foo[1]}", new NumberFieldType("foo", NumberType.LONG)), + equalTo(List.of(10L, 20L, 100L, 200L)) + ); + } + + @Override + protected ScriptContext scriptContext() { + return LongArrayScriptFieldScript.CONTEXT; + } + + @Override + protected LongArrayScriptFieldScript.LeafFactory newLeafFactory( + LongArrayScriptFieldScript.Factory factory, + Map params, + SourceLookup source, + DocLookup fieldData + ) { + return factory.newFactory(params, source, fieldData); + } + + @Override + protected LongArrayScriptFieldScript newInstance(LongArrayScriptFieldScript.LeafFactory leafFactory, LeafReaderContext context) + throws IOException { + return leafFactory.newInstance(context); + } + + @Override + protected void collect(LongArrayScriptFieldScript script, List result) { + for (long l : script.execute()) { + result.add(l); + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java new file mode 100644 index 0000000000000..e10a5c822f07a --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType; +import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.search.lookup.DocLookup; +import org.elasticsearch.search.lookup.SourceLookup; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; + +public class LongScriptFieldScriptTests extends ScriptFieldScriptTestCase< + LongScriptFieldScript, + LongScriptFieldScript.Factory, + LongScriptFieldScript.LeafFactory, + Long> { + public void testConstant() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); + }; + assertThat(execute(indexBuilder, "10"), equalTo(List.of(10L, 10L))); + } + + public void testSource() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 10}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 100}")))); + }; + assertThat(execute(indexBuilder, "source['foo']"), equalTo(List.of(10L, 100L))); + } + + public void testDocValues() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", 10))); + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", 100))); + }; + assertThat(execute(indexBuilder, "doc['foo'].value", new NumberFieldType("foo", NumberType.LONG)), equalTo(List.of(10L, 100L))); + } + + @Override + protected ScriptContext scriptContext() { + return LongScriptFieldScript.CONTEXT; + } + + @Override + protected LongScriptFieldScript.LeafFactory newLeafFactory( + LongScriptFieldScript.Factory factory, + Map params, + SourceLookup source, + DocLookup fieldData + ) { + return factory.newFactory(params, source, fieldData); + } + + @Override + protected LongScriptFieldScript newInstance(LongScriptFieldScript.LeafFactory leafFactory, LeafReaderContext context) + throws IOException { + return leafFactory.newInstance(context); + } + + @Override + protected void collect(LongScriptFieldScript script, List result) { + result.add(script.execute()); + } +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java new file mode 100644 index 0000000000000..5285f2af8fe65 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java @@ -0,0 +1,117 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.search.Collector; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.LeafCollector; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Scorable; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.store.Directory; +import org.elasticsearch.Version; +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.painless.PainlessPlugin; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptModule; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.search.lookup.DocLookup; +import org.elasticsearch.search.lookup.SourceLookup; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public abstract class ScriptFieldScriptTestCase extends ESTestCase { + protected abstract ScriptContext scriptContext(); + + protected abstract LF newLeafFactory(F factory, Map params, SourceLookup source, DocLookup fieldData); + + protected abstract S newInstance(LF leafFactory, LeafReaderContext context) throws IOException; + + protected abstract void collect(S script, List result); + + protected final List execute(CheckedConsumer indexBuilder, String script, MappedFieldType... types) + throws IOException { + + // TODO replace painless with mock script engine + ScriptModule scriptModule = new ScriptModule(Settings.EMPTY, List.of(new PainlessPlugin(), new RuntimeFields())); + Map params = new HashMap<>(); + SourceLookup source = new SourceLookup(); + MapperService mapperService = mock(MapperService.class); + for (MappedFieldType type : types) { + when(mapperService.fieldType(type.name())).thenReturn(type); + } + Function> fieldDataLookup = ft -> ft.fielddataBuilder("test") + .build(indexSettings(), ft, null, new NoneCircuitBreakerService(), mapperService); + DocLookup fieldData = new DocLookup(mapperService, fieldDataLookup); + try (ScriptService scriptService = new ScriptService(Settings.EMPTY, scriptModule.engines, scriptModule.contexts)) { + F factory = scriptService.compile(new Script(script), scriptContext()); + + try (Directory directory = newDirectory(); RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { + indexBuilder.accept(indexWriter); + try (DirectoryReader reader = indexWriter.getReader()) { + IndexSearcher searcher = newSearcher(reader); + LF leafFactory = newLeafFactory(factory, params, source, fieldData); + List result = new ArrayList<>(); + searcher.search(new MatchAllDocsQuery(), new Collector() { + @Override + public ScoreMode scoreMode() { + return ScoreMode.COMPLETE_NO_SCORES; + } + + @Override + public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { + S compiled = newInstance(leafFactory, context); + return new LeafCollector() { + @Override + public void setScorer(Scorable scorer) throws IOException {} + + @Override + public void collect(int doc) throws IOException { + compiled.setDocument(doc); + ScriptFieldScriptTestCase.this.collect(compiled, result); + } + }; + } + }); + return result; + } + } + } + } + + private IndexSettings indexSettings() { + return new IndexSettings( + IndexMetadata.builder("_index") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)) + .numberOfShards(1) + .numberOfReplicas(0) + .creationDate(System.currentTimeMillis()) + .build(), + Settings.EMPTY + ); + } +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScriptTests.java new file mode 100644 index 0000000000000..55198ce968469 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScriptTests.java @@ -0,0 +1,80 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.search.lookup.DocLookup; +import org.elasticsearch.search.lookup.SourceLookup; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; + +public class StringScriptFieldsScriptTests extends ScriptFieldScriptTestCase< + StringScriptFieldsScript, + StringScriptFieldsScript.Factory, + StringScriptFieldsScript.LeafFactory, + String> { + + public void testConstant() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef(randomAlphaOfLength(2))))); + iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef(randomAlphaOfLength(2))))); + }; + assertThat(execute(indexBuilder, "\"cat\""), equalTo(List.of("cat", "cat"))); + } + + public void testSource() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": \"cat\"}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": \"dog\"}")))); + }; + assertThat(execute(indexBuilder, "source['foo']"), equalTo(List.of("cat", "dog"))); + } + + public void testDocValues() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef("cat")))); + iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef("dog")))); + }; + assertThat(execute(indexBuilder, "doc['foo'].value", new KeywordFieldType("foo")), equalTo(List.of("cat", "dog"))); + } + + @Override + protected ScriptContext scriptContext() { + return StringScriptFieldsScript.CONTEXT; + } + + @Override + protected StringScriptFieldsScript.LeafFactory newLeafFactory( + StringScriptFieldsScript.Factory factory, + Map params, + SourceLookup source, + DocLookup fieldData + ) { + return factory.newFactory(params, source, fieldData); + } + + @Override + protected StringScriptFieldsScript newInstance(StringScriptFieldsScript.LeafFactory leafFactory, LeafReaderContext context) throws IOException { + return leafFactory.newInstance(context); + } + + @Override + protected void collect(StringScriptFieldsScript script, List result) { + result.add(script.execute()); + } +} From 76d8b94d873984682964f5d6c87047e7e93e4fd8 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 18 Jun 2020 16:20:40 -0400 Subject: [PATCH 03/35] Scripted fields: rework script contexts (#58342) This reworks the script contexts to support multiple values for each hit. Just like before, these are almost certainly not the final implementation, but they give us something to iterate on. --- x-pack/plugin/runtime-fields/build.gradle | 2 +- .../AbstractScriptFieldsScript.java | 11 +- .../LongArrayScriptFieldScript.java | 34 ----- .../runtimefields/LongScriptFieldScript.java | 35 +++++- .../xpack/runtimefields/RuntimeFields.java | 3 +- .../RuntimeFieldsPainlessExtension.java | 24 ++++ .../StringScriptFieldScript.java | 62 ++++++++++ .../StringScriptFieldsScript.java | 34 ----- ...asticsearch.painless.spi.PainlessExtension | 1 + .../xpack/runtimefields/long_whitelist.txt | 14 +++ .../xpack/runtimefields/string_whitelist.txt | 14 +++ .../LongArrayScriptFieldScriptTests.java | 87 ------------- .../LongScriptFieldScriptTests.java | 49 ++++++-- .../ScriptFieldScriptTestCase.java | 13 +- .../StringScriptFieldScriptTests.java | 116 ++++++++++++++++++ .../StringScriptFieldsScriptTests.java | 80 ------------ 16 files changed, 315 insertions(+), 264 deletions(-) delete mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScript.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsPainlessExtension.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java delete mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScript.java create mode 100644 x-pack/plugin/runtime-fields/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension create mode 100644 x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/long_whitelist.txt create mode 100644 x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/string_whitelist.txt delete mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScriptTests.java create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java delete mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScriptTests.java diff --git a/x-pack/plugin/runtime-fields/build.gradle b/x-pack/plugin/runtime-fields/build.gradle index b1fbb6b839f63..ebba3ddfffed6 100644 --- a/x-pack/plugin/runtime-fields/build.gradle +++ b/x-pack/plugin/runtime-fields/build.gradle @@ -6,7 +6,7 @@ esplugin { name 'x-pack-runtime-fields' description 'A module which adds support for runtime fields' classname 'org.elasticsearch.xpack.runtimefields.RuntimeFields' - extendedPlugins = ['x-pack-core'] + extendedPlugins = ['x-pack-core', 'lang-painless'] } archivesBaseName = 'x-pack-runtime-fields' diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldsScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldsScript.java index 95c419b6ad558..952192c52dbf7 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldsScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldsScript.java @@ -35,7 +35,7 @@ public AbstractScriptFieldsScript(Map params, SourceLookup sourc /** * Set the document to run the script against. */ - public void setDocument(int docId) { + public final void setDocument(int docId) { source.setSegmentAndDocument(ctx, docId); fieldData.setDocument(docId); } @@ -43,21 +43,24 @@ public void setDocument(int docId) { /** * Expose the {@code params} of the script to the script itself. */ - public Map getParams() { + public final Map getParams() { return params; } /** * Expose the {@code _source} to the script. */ - public Map getSource() { + public final Map getSource() { return source; } /** * Expose field data to the script as {@code doc}. */ - public Map> getDoc() { + public final Map> getDoc() { return fieldData; } + + public abstract void execute(); + } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScript.java deleted file mode 100644 index e9676f580b45f..0000000000000 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScript.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.runtimefields; - -import org.apache.lucene.index.LeafReaderContext; -import org.elasticsearch.script.ScriptContext; -import org.elasticsearch.script.ScriptFactory; -import org.elasticsearch.search.lookup.DocLookup; -import org.elasticsearch.search.lookup.SourceLookup; - -import java.io.IOException; -import java.util.Map; - -public abstract class LongArrayScriptFieldScript extends AbstractScriptFieldsScript { - public static final ScriptContext CONTEXT = new ScriptContext<>("long_array_script_field", Factory.class); - public static final String[] PARAMETERS = {}; - - public interface Factory extends ScriptFactory { - LeafFactory newFactory(Map params, SourceLookup source, DocLookup fieldData); - } - public static interface LeafFactory { - LongArrayScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException; - } - - public LongArrayScriptFieldScript(Map params, SourceLookup source, DocLookup fieldData, LeafReaderContext ctx) { - super(params, source, fieldData, ctx); - } - - public abstract long[] execute(); -} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java index 4e1dce5c9facc..aad1aa45fcac5 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java @@ -7,28 +7,55 @@ package org.elasticsearch.xpack.runtimefields; import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptFactory; import org.elasticsearch.search.lookup.DocLookup; import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; +import java.util.List; import java.util.Map; +import java.util.function.LongConsumer; public abstract class LongScriptFieldScript extends AbstractScriptFieldsScript { - public static final ScriptContext CONTEXT = new ScriptContext<>("long_script_field", Factory.class); + static final ScriptContext CONTEXT = new ScriptContext<>("long_script_field", Factory.class); + static List whitelist() { + return List.of(WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "long_whitelist.txt")); + } + public static final String[] PARAMETERS = {}; public interface Factory extends ScriptFactory { LeafFactory newFactory(Map params, SourceLookup source, DocLookup fieldData); } public static interface LeafFactory { - LongScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException; + LongScriptFieldScript newInstance(LeafReaderContext ctx, LongConsumer sync) throws IOException; } - public LongScriptFieldScript(Map params, SourceLookup source, DocLookup fieldData, LeafReaderContext ctx) { + private final LongConsumer sync; + + public LongScriptFieldScript( + Map params, + SourceLookup source, + DocLookup fieldData, + LeafReaderContext ctx, + LongConsumer sync + ) { super(params, source, fieldData, ctx); + this.sync = sync; } - public abstract long execute(); + public static class Value { + private final LongScriptFieldScript script; + + public Value(LongScriptFieldScript script) { + this.script = script; + } + + public void value(long v) { + script.sync.accept(v); + } + } } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java index 0a91bd6db2021..e15d9b4bc3b18 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java @@ -26,9 +26,8 @@ public Map getMappers() { @Override public List> getContexts() { return List.of( - LongArrayScriptFieldScript.CONTEXT, LongScriptFieldScript.CONTEXT, - StringScriptFieldsScript.CONTEXT + StringScriptFieldScript.CONTEXT ); } } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsPainlessExtension.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsPainlessExtension.java new file mode 100644 index 0000000000000..5757b6667e29c --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsPainlessExtension.java @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.elasticsearch.painless.spi.PainlessExtension; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.script.ScriptContext; + +import java.util.List; +import java.util.Map; + +public class RuntimeFieldsPainlessExtension implements PainlessExtension { + @Override + public Map, List> getContextWhitelists() { + return Map.of( + LongScriptFieldScript.CONTEXT, LongScriptFieldScript.whitelist(), + StringScriptFieldScript.CONTEXT, StringScriptFieldScript.whitelist() + ); + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java new file mode 100644 index 0000000000000..3e709972a1af6 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptFactory; +import org.elasticsearch.search.lookup.DocLookup; +import org.elasticsearch.search.lookup.SourceLookup; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.function.Consumer; + +public abstract class StringScriptFieldScript extends AbstractScriptFieldsScript { + static final ScriptContext CONTEXT = new ScriptContext<>("string_script_field", Factory.class); + static List whitelist() { + return List.of(WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "string_whitelist.txt")); + } + + public static final String[] PARAMETERS = {}; + + public interface Factory extends ScriptFactory { + LeafFactory newFactory(Map params, SourceLookup source, DocLookup fieldData); + } + + public static interface LeafFactory { + StringScriptFieldScript newInstance(LeafReaderContext ctx, Consumer sync) throws IOException; + } + + private final Consumer sync; + + public StringScriptFieldScript( + Map params, + SourceLookup source, + DocLookup fieldData, + LeafReaderContext ctx, + Consumer sync + ) { + super(params, source, fieldData, ctx); + this.sync = sync; + } + + public static class Value { + private final StringScriptFieldScript script; + + public Value(StringScriptFieldScript script) { + this.script = script; + } + + public void value(String v) { + script.sync.accept(v); + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScript.java deleted file mode 100644 index b8f3809de3af3..0000000000000 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScript.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.runtimefields; - -import org.apache.lucene.index.LeafReaderContext; -import org.elasticsearch.script.ScriptContext; -import org.elasticsearch.script.ScriptFactory; -import org.elasticsearch.search.lookup.DocLookup; -import org.elasticsearch.search.lookup.SourceLookup; - -import java.io.IOException; -import java.util.Map; - -public abstract class StringScriptFieldsScript extends AbstractScriptFieldsScript { - public static final ScriptContext CONTEXT = new ScriptContext<>("string_script_field", Factory.class); - public static final String[] PARAMETERS = {}; - - public interface Factory extends ScriptFactory { - LeafFactory newFactory(Map params, SourceLookup source, DocLookup fieldData); - } - public static interface LeafFactory { - StringScriptFieldsScript newInstance(LeafReaderContext ctx) throws IOException; - } - - public StringScriptFieldsScript(Map params, SourceLookup source, DocLookup fieldData, LeafReaderContext ctx) { - super(params, source, fieldData, ctx); - } - - public abstract String execute(); -} diff --git a/x-pack/plugin/runtime-fields/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension b/x-pack/plugin/runtime-fields/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension new file mode 100644 index 0000000000000..cd804b8d8f6c9 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/resources/META-INF/services/org.elasticsearch.painless.spi.PainlessExtension @@ -0,0 +1 @@ +org.elasticsearch.xpack.runtimefields.RuntimeFieldsPainlessExtension \ No newline at end of file diff --git a/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/long_whitelist.txt b/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/long_whitelist.txt new file mode 100644 index 0000000000000..630c679bc21cc --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/long_whitelist.txt @@ -0,0 +1,14 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +# The whitelist for long-valued runtime fields + +class org.elasticsearch.xpack.runtimefields.LongScriptFieldScript @no_import { +} + +static_import { + void value(org.elasticsearch.xpack.runtimefields.LongScriptFieldScript, long) bound_to org.elasticsearch.xpack.runtimefields.LongScriptFieldScript$Value +} diff --git a/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/string_whitelist.txt b/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/string_whitelist.txt new file mode 100644 index 0000000000000..e0c4e367850f3 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/string_whitelist.txt @@ -0,0 +1,14 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +# The whitelist for long-valued runtime fields + +class org.elasticsearch.xpack.runtimefields.StringScriptFieldScript @no_import { +} + +static_import { + void value(org.elasticsearch.xpack.runtimefields.StringScriptFieldScript, String) bound_to org.elasticsearch.xpack.runtimefields.StringScriptFieldScript$Value +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScriptTests.java deleted file mode 100644 index 6a3114bd15698..0000000000000 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongArrayScriptFieldScriptTests.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.runtimefields; - -import org.apache.lucene.document.SortedNumericDocValuesField; -import org.apache.lucene.document.StoredField; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.CheckedConsumer; -import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType; -import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; -import org.elasticsearch.script.ScriptContext; -import org.elasticsearch.search.lookup.DocLookup; -import org.elasticsearch.search.lookup.SourceLookup; - -import java.io.IOException; -import java.util.List; -import java.util.Map; - -import static org.hamcrest.Matchers.equalTo; - -public class LongArrayScriptFieldScriptTests extends ScriptFieldScriptTestCase< - LongArrayScriptFieldScript, - LongArrayScriptFieldScript.Factory, - LongArrayScriptFieldScript.LeafFactory, - Long> { - - public void testConstant() throws IOException { - CheckedConsumer indexBuilder = iw -> { - iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); - iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); - }; - assertThat(execute(indexBuilder, "new long[] {10, 100}"), equalTo(List.of(10L, 100L, 10L, 100L))); - } - - public void testSource() throws IOException { - CheckedConsumer indexBuilder = iw -> { - iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 10, \"bar\": 20}")))); - iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 100, \"bar\": 200}")))); - }; - assertThat(execute(indexBuilder, "new long[] {source['foo'], source['bar']}"), equalTo(List.of(10L, 20L, 100L, 200L))); - } - - public void testDocValues() throws IOException { - CheckedConsumer indexBuilder = iw -> { - iw.addDocument(List.of(new SortedNumericDocValuesField("foo", 10), new SortedNumericDocValuesField("foo", 20))); - iw.addDocument(List.of(new SortedNumericDocValuesField("foo", 100), new SortedNumericDocValuesField("foo", 200))); - }; - assertThat( - execute(indexBuilder, "def foo = doc['foo']; new long[] {foo[0], foo[1]}", new NumberFieldType("foo", NumberType.LONG)), - equalTo(List.of(10L, 20L, 100L, 200L)) - ); - } - - @Override - protected ScriptContext scriptContext() { - return LongArrayScriptFieldScript.CONTEXT; - } - - @Override - protected LongArrayScriptFieldScript.LeafFactory newLeafFactory( - LongArrayScriptFieldScript.Factory factory, - Map params, - SourceLookup source, - DocLookup fieldData - ) { - return factory.newFactory(params, source, fieldData); - } - - @Override - protected LongArrayScriptFieldScript newInstance(LongArrayScriptFieldScript.LeafFactory leafFactory, LeafReaderContext context) - throws IOException { - return leafFactory.newInstance(context); - } - - @Override - protected void collect(LongArrayScriptFieldScript script, List result) { - for (long l : script.execute()) { - result.add(l); - } - } -} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java index e10a5c822f07a..05d48808b5620 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java @@ -29,28 +29,59 @@ public class LongScriptFieldScriptTests extends ScriptFieldScriptTestCase< LongScriptFieldScript.Factory, LongScriptFieldScript.LeafFactory, Long> { + public void testConstant() throws IOException { CheckedConsumer indexBuilder = iw -> { iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); }; - assertThat(execute(indexBuilder, "10"), equalTo(List.of(10L, 10L))); + assertThat(execute(indexBuilder, "value(10)"), equalTo(List.of(10L, 10L))); + } + + public void testTwoConstants() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); + }; + assertThat(execute(indexBuilder, "value(10); value(20)"), equalTo(List.of(10L, 20L, 10L, 20L))); } public void testSource() throws IOException { CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 1}")))); iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 10}")))); - iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 100}")))); }; - assertThat(execute(indexBuilder, "source['foo']"), equalTo(List.of(10L, 100L))); + assertThat(execute(indexBuilder, "value(source['foo'] * 10)"), equalTo(List.of(10L, 100L))); + } + + public void testTwoSourceFields() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 1, \"bar\": 2}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 10, \"bar\": 20}")))); + }; + assertThat(execute(indexBuilder, "value(source['foo'] * 10); value(source['bar'] * 10)"), equalTo(List.of(10L, 20L, 100L, 200L))); } public void testDocValues() throws IOException { CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", 1))); iw.addDocument(List.of(new SortedNumericDocValuesField("foo", 10))); - iw.addDocument(List.of(new SortedNumericDocValuesField("foo", 100))); }; - assertThat(execute(indexBuilder, "doc['foo'].value", new NumberFieldType("foo", NumberType.LONG)), equalTo(List.of(10L, 100L))); + assertThat( + execute(indexBuilder, "value(doc['foo'].value * 10)", new NumberFieldType("foo", NumberType.LONG)), + equalTo(List.of(10L, 100L)) + ); + } + + public void testTwoDocValuesValues() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", 1), new SortedNumericDocValuesField("foo", 2))); + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", 10), new SortedNumericDocValuesField("foo", 20))); + }; + assertThat( + execute(indexBuilder, "for (long l : doc['foo']) {value(l * 10)}", new NumberFieldType("foo", NumberType.LONG)), + equalTo(List.of(10L, 20L, 100L, 200L)) + ); } @Override @@ -69,13 +100,9 @@ protected LongScriptFieldScript.LeafFactory newLeafFactory( } @Override - protected LongScriptFieldScript newInstance(LongScriptFieldScript.LeafFactory leafFactory, LeafReaderContext context) + protected LongScriptFieldScript newInstance(LongScriptFieldScript.LeafFactory leafFactory, LeafReaderContext context, List result) throws IOException { - return leafFactory.newInstance(context); - } - @Override - protected void collect(LongScriptFieldScript script, List result) { - result.add(script.execute()); + return leafFactory.newInstance(context, result::add); } } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java index 5285f2af8fe65..c5c40ffa0972b 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java @@ -49,15 +49,14 @@ public abstract class ScriptFieldScriptTestCase params, SourceLookup source, DocLookup fieldData); - protected abstract S newInstance(LF leafFactory, LeafReaderContext context) throws IOException; - - protected abstract void collect(S script, List result); + protected abstract S newInstance(LF leafFactory, LeafReaderContext context, List results) throws IOException; protected final List execute(CheckedConsumer indexBuilder, String script, MappedFieldType... types) throws IOException { - // TODO replace painless with mock script engine - ScriptModule scriptModule = new ScriptModule(Settings.EMPTY, List.of(new PainlessPlugin(), new RuntimeFields())); + PainlessPlugin painlessPlugin = new PainlessPlugin(); + painlessPlugin.reloadSPI(Thread.currentThread().getContextClassLoader()); + ScriptModule scriptModule = new ScriptModule(Settings.EMPTY, List.of(painlessPlugin, new RuntimeFields())); Map params = new HashMap<>(); SourceLookup source = new SourceLookup(); MapperService mapperService = mock(MapperService.class); @@ -84,7 +83,7 @@ public ScoreMode scoreMode() { @Override public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { - S compiled = newInstance(leafFactory, context); + S compiled = newInstance(leafFactory, context, result); return new LeafCollector() { @Override public void setScorer(Scorable scorer) throws IOException {} @@ -92,7 +91,7 @@ public void setScorer(Scorable scorer) throws IOException {} @Override public void collect(int doc) throws IOException { compiled.setDocument(doc); - ScriptFieldScriptTestCase.this.collect(compiled, result); + compiled.execute(); } }; } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java new file mode 100644 index 0000000000000..658001f743281 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java @@ -0,0 +1,116 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.document.SortedSetDocValuesField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.search.lookup.DocLookup; +import org.elasticsearch.search.lookup.SourceLookup; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; + +public class StringScriptFieldScriptTests extends ScriptFieldScriptTestCase< + StringScriptFieldScript, + StringScriptFieldScript.Factory, + StringScriptFieldScript.LeafFactory, + String> { + + public void testConstant() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef(randomAlphaOfLength(2))))); + iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef(randomAlphaOfLength(2))))); + }; + assertThat(execute(indexBuilder, "value('cat')"), equalTo(List.of("cat", "cat"))); + } + + public void testTwoConstants() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef(randomAlphaOfLength(2))))); + iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef(randomAlphaOfLength(2))))); + }; + assertThat(execute(indexBuilder, "value('cat'); value('dog')"), equalTo(List.of("cat", "dog", "cat", "dog"))); + } + + public void testSource() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": \"cat\"}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": \"dog\"}")))); + }; + assertThat(execute(indexBuilder, "value(source['foo'] + 'o')"), equalTo(List.of("cato", "dogo"))); + } + + public void testTwoSourceFields() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": \"cat\", \"bar\": \"chicken\"}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": \"dog\", \"bar\": \"pig\"}")))); + }; + assertThat( + execute(indexBuilder, "value(source['foo'] + 'o'); value(source['bar'] + 'ie')"), + equalTo(List.of("cato", "chickenie", "dogo", "pigie")) + ); + } + + public void testDocValues() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef("cat")))); + iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef("dog")))); + }; + assertThat(execute(indexBuilder, "value(doc['foo'].value + 'o')", new KeywordFieldType("foo")), equalTo(List.of("cato", "dogo"))); + } + + public void testTwoDocValuesValues() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument( + List.of( + new SortedSetDocValuesField("foo", new BytesRef("cat")), + new SortedSetDocValuesField("foo", new BytesRef("chicken")) + ) + ); + iw.addDocument( + List.of(new SortedSetDocValuesField("foo", new BytesRef("dog")), new SortedSetDocValuesField("foo", new BytesRef("pig"))) + ); + }; + assertThat( + execute(indexBuilder, "for (String s: doc['foo']) {value(s + 'o')}", new KeywordFieldType("foo")), + equalTo(List.of("cato", "chickeno", "dogo", "pigo")) + ); + } + + @Override + protected ScriptContext scriptContext() { + return StringScriptFieldScript.CONTEXT; + } + + @Override + protected StringScriptFieldScript.LeafFactory newLeafFactory( + StringScriptFieldScript.Factory factory, + Map params, + SourceLookup source, + DocLookup fieldData + ) { + return factory.newFactory(params, source, fieldData); + } + + @Override + protected StringScriptFieldScript newInstance( + StringScriptFieldScript.LeafFactory leafFactory, + LeafReaderContext context, + List result + ) throws IOException { + return leafFactory.newInstance(context, result::add); + } +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScriptTests.java deleted file mode 100644 index 55198ce968469..0000000000000 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldsScriptTests.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.runtimefields; - -import org.apache.lucene.document.SortedSetDocValuesField; -import org.apache.lucene.document.StoredField; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.CheckedConsumer; -import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType; -import org.elasticsearch.script.ScriptContext; -import org.elasticsearch.search.lookup.DocLookup; -import org.elasticsearch.search.lookup.SourceLookup; - -import java.io.IOException; -import java.util.List; -import java.util.Map; - -import static org.hamcrest.Matchers.equalTo; - -public class StringScriptFieldsScriptTests extends ScriptFieldScriptTestCase< - StringScriptFieldsScript, - StringScriptFieldsScript.Factory, - StringScriptFieldsScript.LeafFactory, - String> { - - public void testConstant() throws IOException { - CheckedConsumer indexBuilder = iw -> { - iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef(randomAlphaOfLength(2))))); - iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef(randomAlphaOfLength(2))))); - }; - assertThat(execute(indexBuilder, "\"cat\""), equalTo(List.of("cat", "cat"))); - } - - public void testSource() throws IOException { - CheckedConsumer indexBuilder = iw -> { - iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": \"cat\"}")))); - iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": \"dog\"}")))); - }; - assertThat(execute(indexBuilder, "source['foo']"), equalTo(List.of("cat", "dog"))); - } - - public void testDocValues() throws IOException { - CheckedConsumer indexBuilder = iw -> { - iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef("cat")))); - iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef("dog")))); - }; - assertThat(execute(indexBuilder, "doc['foo'].value", new KeywordFieldType("foo")), equalTo(List.of("cat", "dog"))); - } - - @Override - protected ScriptContext scriptContext() { - return StringScriptFieldsScript.CONTEXT; - } - - @Override - protected StringScriptFieldsScript.LeafFactory newLeafFactory( - StringScriptFieldsScript.Factory factory, - Map params, - SourceLookup source, - DocLookup fieldData - ) { - return factory.newFactory(params, source, fieldData); - } - - @Override - protected StringScriptFieldsScript newInstance(StringScriptFieldsScript.LeafFactory leafFactory, LeafReaderContext context) throws IOException { - return leafFactory.newInstance(context); - } - - @Override - protected void collect(StringScriptFieldsScript script, List result) { - result.add(script.execute()); - } -} From 818403b4e814f7e493d33e4507fc356916888e8b Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 18 Jun 2020 15:29:00 -0400 Subject: [PATCH 04/35] Spotless and maybe tests? --- ...pt.java => AbstractScriptFieldScript.java} | 4 +- .../DoubleScriptFieldScript.java | 63 +++++++++ .../runtimefields/LongScriptFieldScript.java | 6 +- .../xpack/runtimefields/RuntimeFields.java | 5 +- .../RuntimeFieldsPainlessExtension.java | 7 +- .../StringScriptFieldScript.java | 5 +- .../xpack/runtimefields/double_whitelist.txt | 14 ++ .../DoubleScriptFieldScriptTests.java | 124 ++++++++++++++++++ .../ScriptFieldScriptTestCase.java | 8 +- 9 files changed, 221 insertions(+), 15 deletions(-) rename x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/{AbstractScriptFieldsScript.java => AbstractScriptFieldScript.java} (90%) create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java create mode 100644 x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/double_whitelist.txt create mode 100644 x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldsScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldScript.java similarity index 90% rename from x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldsScript.java rename to x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldScript.java index 952192c52dbf7..b9d605c796530 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldsScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldScript.java @@ -19,13 +19,13 @@ * Abstract base for scripts to execute to build scripted fields. Inspired by * {@link AggregationScript} but hopefully with less historical baggage. */ -public abstract class AbstractScriptFieldsScript { +public abstract class AbstractScriptFieldScript { private final Map params; private final LeafReaderContext ctx; private final SourceLookup source; private final LeafDocLookup fieldData; - public AbstractScriptFieldsScript(Map params, SourceLookup source, DocLookup fieldData, LeafReaderContext ctx) { + public AbstractScriptFieldScript(Map params, SourceLookup source, DocLookup fieldData, LeafReaderContext ctx) { this.params = params; this.source = source; this.fieldData = fieldData.getLeafDocLookup(ctx); diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java new file mode 100644 index 0000000000000..e86bfb20f71b6 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptFactory; +import org.elasticsearch.search.lookup.DocLookup; +import org.elasticsearch.search.lookup.SourceLookup; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.function.DoubleConsumer; + +public abstract class DoubleScriptFieldScript extends AbstractScriptFieldScript { + static final ScriptContext CONTEXT = new ScriptContext<>("double_script_field", Factory.class); + + static List whitelist() { + return List.of(WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "double_whitelist.txt")); + } + + public static final String[] PARAMETERS = {}; + + public interface Factory extends ScriptFactory { + LeafFactory newFactory(Map params, SourceLookup source, DocLookup fieldData); + } + + public interface LeafFactory { + DoubleScriptFieldScript newInstance(LeafReaderContext ctx, DoubleConsumer sync) throws IOException; + } + + private final DoubleConsumer sync; + + public DoubleScriptFieldScript( + Map params, + SourceLookup source, + DocLookup fieldData, + LeafReaderContext ctx, + DoubleConsumer sync + ) { + super(params, source, fieldData, ctx); + this.sync = sync; + } + + public static class Value { + private final DoubleScriptFieldScript script; + + public Value(DoubleScriptFieldScript script) { + this.script = script; + } + + public void value(double v) { + script.sync.accept(v); + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java index aad1aa45fcac5..e99fa4a944ba3 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java @@ -19,8 +19,9 @@ import java.util.Map; import java.util.function.LongConsumer; -public abstract class LongScriptFieldScript extends AbstractScriptFieldsScript { +public abstract class LongScriptFieldScript extends AbstractScriptFieldScript { static final ScriptContext CONTEXT = new ScriptContext<>("long_script_field", Factory.class); + static List whitelist() { return List.of(WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "long_whitelist.txt")); } @@ -30,7 +31,8 @@ static List whitelist() { public interface Factory extends ScriptFactory { LeafFactory newFactory(Map params, SourceLookup source, DocLookup fieldData); } - public static interface LeafFactory { + + public interface LeafFactory { LongScriptFieldScript newInstance(LeafReaderContext ctx, LongConsumer sync) throws IOException; } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java index e15d9b4bc3b18..6b97b0dcb795c 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java @@ -25,9 +25,6 @@ public Map getMappers() { @Override public List> getContexts() { - return List.of( - LongScriptFieldScript.CONTEXT, - StringScriptFieldScript.CONTEXT - ); + return List.of(DoubleScriptFieldScript.CONTEXT, LongScriptFieldScript.CONTEXT, StringScriptFieldScript.CONTEXT); } } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsPainlessExtension.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsPainlessExtension.java index 5757b6667e29c..0ccbeee8069a1 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsPainlessExtension.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsPainlessExtension.java @@ -16,9 +16,10 @@ public class RuntimeFieldsPainlessExtension implements PainlessExtension { @Override public Map, List> getContextWhitelists() { - return Map.of( - LongScriptFieldScript.CONTEXT, LongScriptFieldScript.whitelist(), - StringScriptFieldScript.CONTEXT, StringScriptFieldScript.whitelist() + return Map.ofEntries( + Map.entry(DoubleScriptFieldScript.CONTEXT, DoubleScriptFieldScript.whitelist()), + Map.entry(LongScriptFieldScript.CONTEXT, LongScriptFieldScript.whitelist()), + Map.entry(StringScriptFieldScript.CONTEXT, StringScriptFieldScript.whitelist()) ); } } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java index 3e709972a1af6..2556381981236 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java @@ -19,8 +19,9 @@ import java.util.Map; import java.util.function.Consumer; -public abstract class StringScriptFieldScript extends AbstractScriptFieldsScript { +public abstract class StringScriptFieldScript extends AbstractScriptFieldScript { static final ScriptContext CONTEXT = new ScriptContext<>("string_script_field", Factory.class); + static List whitelist() { return List.of(WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "string_whitelist.txt")); } @@ -31,7 +32,7 @@ public interface Factory extends ScriptFactory { LeafFactory newFactory(Map params, SourceLookup source, DocLookup fieldData); } - public static interface LeafFactory { + public interface LeafFactory { StringScriptFieldScript newInstance(LeafReaderContext ctx, Consumer sync) throws IOException; } diff --git a/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/double_whitelist.txt b/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/double_whitelist.txt new file mode 100644 index 0000000000000..55a177f21579c --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/double_whitelist.txt @@ -0,0 +1,14 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +# The whitelist for long-valued runtime fields + +class org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript @no_import { +} + +static_import { + void value(org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript, double) bound_to org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript$Value +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java new file mode 100644 index 0000000000000..37c6c5d35ce41 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java @@ -0,0 +1,124 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.NumericUtils; +import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType; +import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.search.lookup.DocLookup; +import org.elasticsearch.search.lookup.SourceLookup; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; + +public class DoubleScriptFieldScriptTests extends ScriptFieldScriptTestCase< + DoubleScriptFieldScript, + DoubleScriptFieldScript.Factory, + DoubleScriptFieldScript.LeafFactory, + Double> { + + public void testConstant() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(randomDouble())))); + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(randomDouble())))); + }; + assertThat(execute(indexBuilder, "value(3.14)"), equalTo(List.of(3.14, 3.14))); + } + + public void testTwoConstants() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); + }; + assertThat(execute(indexBuilder, "value(3.14); value(2.72)"), equalTo(List.of(3.14, 2.72, 3.14, 2.72))); + } + + public void testSource() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 1}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 10}")))); + }; + assertThat(execute(indexBuilder, "value(source['foo'] * 10.1)"), equalTo(List.of(10.1, 101.0))); + } + + public void testTwoSourceFields() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 1, \"bar\": 2}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 10, \"bar\": 20}")))); + }; + assertThat( + execute(indexBuilder, "value(source['foo'] * 10.1); value(source['bar'] * 10.2)"), + equalTo(List.of(10.1, 20.4, 101.0, 204.0)) + ); + } + + public void testDocValues() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(1.1)))); + iw.addDocument(List.of(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(10.1)))); + }; + assertThat( + execute(indexBuilder, "value(doc['foo'].value * 9.9)", new NumberFieldType("foo", NumberType.DOUBLE)), + equalTo(List.of(10.89, 99.99)) + ); + } + + public void testTwoDocValuesValues() throws IOException { + CheckedConsumer indexBuilder = iw -> { + iw.addDocument( + List.of( + new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(1.1)), + new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(2.2)) + ) + ); + iw.addDocument( + List.of( + new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(10.1)), + new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(20.1)) + ) + ); + }; + assertThat( + execute(indexBuilder, "for (double d : doc['foo']) {value(d * 9.9)}", new NumberFieldType("foo", NumberType.DOUBLE)), + equalTo(List.of(10.89, 21.78, 99.99, 198.99)) + ); + } + + @Override + protected ScriptContext scriptContext() { + return DoubleScriptFieldScript.CONTEXT; + } + + @Override + protected DoubleScriptFieldScript.LeafFactory newLeafFactory( + DoubleScriptFieldScript.Factory factory, + Map params, + SourceLookup source, + DocLookup fieldData + ) { + return factory.newFactory(params, source, fieldData); + } + + @Override + protected DoubleScriptFieldScript newInstance( + DoubleScriptFieldScript.LeafFactory leafFactory, + LeafReaderContext context, + List result + ) throws IOException { + return leafFactory.newInstance(context, result::add); + } +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java index c5c40ffa0972b..2b8fc1bef2fbf 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java @@ -35,6 +35,8 @@ import org.elasticsearch.test.ESTestCase; import java.io.IOException; +import java.security.AccessController; +import java.security.PrivilegedAction; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -44,7 +46,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -public abstract class ScriptFieldScriptTestCase extends ESTestCase { +public abstract class ScriptFieldScriptTestCase extends ESTestCase { protected abstract ScriptContext scriptContext(); protected abstract LF newLeafFactory(F factory, Map params, SourceLookup source, DocLookup fieldData); @@ -67,7 +69,9 @@ protected final List execute(CheckedConsumer .build(indexSettings(), ft, null, new NoneCircuitBreakerService(), mapperService); DocLookup fieldData = new DocLookup(mapperService, fieldDataLookup); try (ScriptService scriptService = new ScriptService(Settings.EMPTY, scriptModule.engines, scriptModule.contexts)) { - F factory = scriptService.compile(new Script(script), scriptContext()); + F factory = AccessController.doPrivileged( + (PrivilegedAction) () -> scriptService.compile(new Script(script), scriptContext()) + ); try (Directory directory = newDirectory(); RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { indexBuilder.accept(indexWriter); From 3669ec43ef95a8ce775a2f7aa8d72f985006eb5a Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 26 Jun 2020 11:00:50 -0400 Subject: [PATCH 05/35] Fixup test case after merge Painless extensions work differently now. --- .../xpack/runtimefields/ScriptFieldScriptTestCase.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java index 2b8fc1bef2fbf..a086953142662 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java @@ -26,6 +26,7 @@ import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.painless.PainlessPlugin; +import org.elasticsearch.plugins.ExtensiblePlugin.ExtensionLoader; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptModule; @@ -57,7 +58,13 @@ protected final List execute(CheckedConsumer throws IOException { PainlessPlugin painlessPlugin = new PainlessPlugin(); - painlessPlugin.reloadSPI(Thread.currentThread().getContextClassLoader()); + painlessPlugin.loadExtensions(new ExtensionLoader() { + @Override + @SuppressWarnings("unchecked") // We only ever load painless extensions here so it is fairly safe. + public List loadExtensions(Class extensionPointType) { + return (List) List.of(new RuntimeFieldsPainlessExtension()); + } + }); ScriptModule scriptModule = new ScriptModule(Settings.EMPTY, List.of(painlessPlugin, new RuntimeFields())); Map params = new HashMap<>(); SourceLookup source = new SourceLookup(); From 3a390b53bf5bd9024a488fffa196e4338b431a0b Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 26 Jun 2020 16:26:26 -0400 Subject: [PATCH 06/35] Get runtime fields tests passing Uses a temporary hack to allow us to use painless in unit tests. --- .../plugin-metadata/plugin-security.policy | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 x-pack/plugin/runtime-fields/src/main/plugin-metadata/plugin-security.policy diff --git a/x-pack/plugin/runtime-fields/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/runtime-fields/src/main/plugin-metadata/plugin-security.policy new file mode 100644 index 0000000000000..7683ec0535e17 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/plugin-metadata/plugin-security.policy @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + + +/* TODO this exists so that we can run painless in unit tests which we + * only want to do because we don't yet have a FieldMapper implementation. + * Once we have that we can move to mock scripts in unit tests and painless + * in integration tests. */ + +grant { + // needed to generate runtime classes + permission java.lang.RuntimePermission "createClassLoader"; + + // needed to find the classloader to load whitelisted classes from + permission java.lang.RuntimePermission "getClassLoader"; +}; + From 59b2a18ad1208c640d874effe90d0f5b102dd395 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Fri, 26 Jun 2020 17:50:14 -0400 Subject: [PATCH 07/35] Add queries --- .../StringScriptFieldScript.java | 190 ++++++++++++++++++ .../ScriptFieldScriptTestCase.java | 19 +- .../StringScriptFieldScriptTests.java | 56 ++++-- 3 files changed, 249 insertions(+), 16 deletions(-) diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java index 2556381981236..19a1edd451039 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java @@ -7,6 +7,20 @@ package org.elasticsearch.xpack.runtimefields; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.PrefixQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.TwoPhaseIterator; +import org.apache.lucene.search.Weight; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.automaton.ByteRunAutomaton; import org.elasticsearch.painless.spi.Whitelist; import org.elasticsearch.painless.spi.WhitelistLoader; import org.elasticsearch.script.ScriptContext; @@ -17,6 +31,8 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.Set; import java.util.function.Consumer; public abstract class StringScriptFieldScript extends AbstractScriptFieldScript { @@ -34,6 +50,14 @@ public interface Factory extends ScriptFactory { public interface LeafFactory { StringScriptFieldScript newInstance(LeafReaderContext ctx, Consumer sync) throws IOException; + + default Query termQuery(String fieldName, String value) { + return new ScriptTermQuery(this, fieldName, value); + } + + default Query prefixQuery(String fieldName, String value) { + return new ScriptPrefixQuery(this, fieldName, value); + } } private final Consumer sync; @@ -60,4 +84,170 @@ public void value(String v) { script.sync.accept(v); } } + + private static class ScriptTermQuery extends Query { + private final LeafFactory leafFactory; + private final String fieldName; + private final String term; + + private ScriptTermQuery(LeafFactory leafFactory, String fieldName, String term) { + this.leafFactory = leafFactory; + this.fieldName = fieldName; + this.term = term; + } + + @Override + public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { + return new ConstantScoreWeight(this, boost) { + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return false; // scripts aren't really cacheable at this point + } + + @Override + public Scorer scorer(LeafReaderContext context) throws IOException { + DocIdSetIterator approximation = DocIdSetIterator.all(context.reader().maxDoc()); + TwoPhaseIterator twoPhase = new TwoPhaseIterator(approximation) { + private boolean hasMatches; + private final StringScriptFieldScript script = leafFactory.newInstance( + context, + s -> { hasMatches = hasMatches || term.equals(s); } + ); + + @Override + public boolean matches() throws IOException { + hasMatches = false; + script.setDocument(approximation.docID()); + script.execute(); + return hasMatches; + } + + @Override + public float matchCost() { + // TODO we have no idea what this should be and no real way to get one + return 1000f; + } + }; + return new ConstantScoreScorer(this, score(), scoreMode, twoPhase); + } + + @Override + public void extractTerms(Set terms) { + terms.add(new Term(fieldName, term)); + } + }; + } + + @Override + public void visit(QueryVisitor visitor) { + visitor.consumeTerms(this, new Term(fieldName, term)); + } + + @Override + public String toString(String field) { + if (fieldName.contentEquals(field)) { + return term; + } + return fieldName + ":" + term; + } + + @Override + public int hashCode() { + return Objects.hash(fieldName, term); + } + + @Override + public boolean equals(Object obj) { + if (obj == null || getClass() != obj.getClass()) { + return false; + } + ScriptTermQuery other = (ScriptTermQuery) obj; + return fieldName.equals(other.fieldName) && term.equals(other.term); + } + } + + private static class ScriptPrefixQuery extends Query { + private final LeafFactory leafFactory; + private final String fieldName; + private final String prefix; + + private ScriptPrefixQuery(LeafFactory leafFactory, String fieldName, String prefix) { + this.leafFactory = leafFactory; + this.fieldName = fieldName; + this.prefix = prefix; + } + + @Override + public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { + return new ConstantScoreWeight(this, boost) { + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return false; // scripts aren't really cacheable at this point + } + + @Override + public Scorer scorer(LeafReaderContext context) throws IOException { + DocIdSetIterator approximation = DocIdSetIterator.all(context.reader().maxDoc()); + TwoPhaseIterator twoPhase = new TwoPhaseIterator(approximation) { + private boolean hasMatches; + private final StringScriptFieldScript script = leafFactory.newInstance( + context, + s -> { hasMatches = hasMatches || s.startsWith(prefix); } + ); + + @Override + public boolean matches() throws IOException { + hasMatches = false; + script.setDocument(approximation.docID()); + script.execute(); + return hasMatches; + } + + @Override + public float matchCost() { + // TODO we have no idea what this should be and no real way to get one + return 1000f; + } + }; + return new ConstantScoreScorer(this, score(), scoreMode, twoPhase); + } + + @Override + public void extractTerms(Set terms) { + // TODO doing this is sort of difficult and maybe not needed. + } + }; + } + + @Override + public void visit(QueryVisitor visitor) { + visitor.consumeTermsMatching( + this, + fieldName, + () -> new ByteRunAutomaton(PrefixQuery.toAutomaton(new BytesRef(prefix)), true, Integer.MAX_VALUE) + ); + } + + @Override + public String toString(String field) { + if (fieldName.contentEquals(field)) { + return prefix + "*"; + } + return fieldName + ":" + prefix + "*"; + } + + @Override + public int hashCode() { + return Objects.hash(fieldName, prefix); + } + + @Override + public boolean equals(Object obj) { + if (obj == null || getClass() != obj.getClass()) { + return false; + } + ScriptPrefixQuery other = (ScriptPrefixQuery) obj; + return fieldName.equals(other.fieldName) && prefix.equals(other.prefix); + } + } } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java index a086953142662..df506d7299eab 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java @@ -13,6 +13,7 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.LeafCollector; import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Query; import org.apache.lucene.search.Scorable; import org.apache.lucene.search.ScoreMode; import org.apache.lucene.store.Directory; @@ -54,8 +55,20 @@ public abstract class ScriptFieldScriptTestCase results) throws IOException; - protected final List execute(CheckedConsumer indexBuilder, String script, MappedFieldType... types) - throws IOException { + protected final List execute( + CheckedConsumer indexBuilder, + String script, + MappedFieldType... types + ) throws IOException { + return execute(indexBuilder, script, f -> new MatchAllDocsQuery(), types); + } + + protected final List execute( + CheckedConsumer indexBuilder, + String script, + Function query, + MappedFieldType... types + ) throws IOException { PainlessPlugin painlessPlugin = new PainlessPlugin(); painlessPlugin.loadExtensions(new ExtensionLoader() { @@ -86,7 +99,7 @@ public List loadExtensions(Class extensionPointType) { IndexSearcher searcher = newSearcher(reader); LF leafFactory = newLeafFactory(factory, params, source, fieldData); List result = new ArrayList<>(); - searcher.search(new MatchAllDocsQuery(), new Collector() { + searcher.search(query.apply(leafFactory), new Collector() { @Override public ScoreMode scoreMode() { return ScoreMode.COMPLETE_NO_SCORES; diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java index 658001f743281..996782236281b 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java @@ -73,20 +73,34 @@ public void testDocValues() throws IOException { } public void testTwoDocValuesValues() throws IOException { - CheckedConsumer indexBuilder = iw -> { - iw.addDocument( - List.of( - new SortedSetDocValuesField("foo", new BytesRef("cat")), - new SortedSetDocValuesField("foo", new BytesRef("chicken")) - ) - ); - iw.addDocument( - List.of(new SortedSetDocValuesField("foo", new BytesRef("dog")), new SortedSetDocValuesField("foo", new BytesRef("pig"))) - ); - }; + assertThat(execute(farmAnimals(), ADD_O, new KeywordFieldType("foo")), equalTo(List.of("cato", "pigo", "chickeno", "dogo"))); + } + + public void testTermQuery() throws IOException { + assertThat(execute(farmAnimals(), ADD_O, lf -> lf.termQuery("foo", "cat"), new KeywordFieldType("foo")), equalTo(List.of())); + assertThat( + execute(farmAnimals(), ADD_O, lf -> lf.termQuery("foo", "cato"), new KeywordFieldType("foo")), + equalTo(List.of("cato", "pigo")) + ); + assertThat( + execute(farmAnimals(), ADD_O, lf -> lf.termQuery("foo", "dogo"), new KeywordFieldType("foo")), + equalTo(List.of("chickeno", "dogo")) + ); + } + + public void testPrefixQuery() throws IOException { + assertThat(execute(farmAnimals(), ADD_O, lf -> lf.prefixQuery("foo", "catdog"), new KeywordFieldType("foo")), equalTo(List.of())); + assertThat( + execute(farmAnimals(), ADD_O, lf -> lf.prefixQuery("foo", "cat"), new KeywordFieldType("foo")), + equalTo(List.of("cato", "pigo")) + ); + assertThat( + execute(farmAnimals(), ADD_O, lf -> lf.prefixQuery("foo", "dogo"), new KeywordFieldType("foo")), + equalTo(List.of("chickeno", "dogo")) + ); assertThat( - execute(indexBuilder, "for (String s: doc['foo']) {value(s + 'o')}", new KeywordFieldType("foo")), - equalTo(List.of("cato", "chickeno", "dogo", "pigo")) + execute(farmAnimals(), ADD_O, lf -> lf.prefixQuery("foo", "c"), new KeywordFieldType("foo")), + equalTo(List.of("cato", "pigo", "chickeno", "dogo")) ); } @@ -113,4 +127,20 @@ protected StringScriptFieldScript newInstance( ) throws IOException { return leafFactory.newInstance(context, result::add); } + + private CheckedConsumer farmAnimals() { + return iw -> { + iw.addDocument( + List.of(new SortedSetDocValuesField("foo", new BytesRef("cat")), new SortedSetDocValuesField("foo", new BytesRef("pig"))) + ); + iw.addDocument( + List.of( + new SortedSetDocValuesField("foo", new BytesRef("chicken")), + new SortedSetDocValuesField("foo", new BytesRef("dog")) + ) + ); + }; + } + + private static final String ADD_O = "for (String s: doc['foo']) {value(s + 'o')}"; } From 28944a4d2db1ec0e1f9995f888699a8eedbc4eb5 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 29 Jun 2020 14:37:48 -0400 Subject: [PATCH 08/35] Better way for queries --- .../DoubleScriptFieldScriptTests.java | 101 +++++++------ .../LongScriptFieldScriptTests.java | 80 +++++----- .../ScriptFieldScriptTestCase.java | 139 ++++++++++-------- .../StringScriptFieldScriptTests.java | 132 +++++++++-------- 4 files changed, 245 insertions(+), 207 deletions(-) diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java index 37c6c5d35ce41..e8446010908fe 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java @@ -8,11 +8,11 @@ import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; -import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; import org.elasticsearch.script.ScriptContext; @@ -32,72 +32,79 @@ public class DoubleScriptFieldScriptTests extends ScriptFieldScriptTestCase< Double> { public void testConstant() throws IOException { - CheckedConsumer indexBuilder = iw -> { - iw.addDocument(List.of(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(randomDouble())))); - iw.addDocument(List.of(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(randomDouble())))); - }; - assertThat(execute(indexBuilder, "value(3.14)"), equalTo(List.of(3.14, 3.14))); + assertThat(randomDoubles().collect("value(3.14)"), equalTo(List.of(3.14, 3.14))); } public void testTwoConstants() throws IOException { - CheckedConsumer indexBuilder = iw -> { - iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); - iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); - }; - assertThat(execute(indexBuilder, "value(3.14); value(2.72)"), equalTo(List.of(3.14, 2.72, 3.14, 2.72))); + assertThat(randomDoubles().collect("value(3.14); value(2.72)"), equalTo(List.of(3.14, 2.72, 3.14, 2.72))); } public void testSource() throws IOException { - CheckedConsumer indexBuilder = iw -> { - iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 1}")))); - iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 10}")))); - }; - assertThat(execute(indexBuilder, "value(source['foo'] * 10.1)"), equalTo(List.of(10.1, 101.0))); + assertThat(singleValueInSource().collect("value(source['foo'] * 10.1)"), equalTo(List.of(10.1, 101.0))); } - public void testTwoSourceFields() throws IOException { - CheckedConsumer indexBuilder = iw -> { - iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 1, \"bar\": 2}")))); - iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 10, \"bar\": 20}")))); - }; + public void testTwoSourceValues() throws IOException { assertThat( - execute(indexBuilder, "value(source['foo'] * 10.1); value(source['bar'] * 10.2)"), + multiValueInSource().collect("value(source['foo'][0] * 10.1); value(source['foo'][1] * 10.2)"), equalTo(List.of(10.1, 20.4, 101.0, 204.0)) ); } public void testDocValues() throws IOException { - CheckedConsumer indexBuilder = iw -> { - iw.addDocument(List.of(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(1.1)))); - iw.addDocument(List.of(new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(10.1)))); - }; - assertThat( - execute(indexBuilder, "value(doc['foo'].value * 9.9)", new NumberFieldType("foo", NumberType.DOUBLE)), - equalTo(List.of(10.89, 99.99)) - ); + assertThat(singleValueInDocValues().collect("value(doc['foo'].value * 9.9)"), equalTo(List.of(10.89, 99.99))); } - public void testTwoDocValuesValues() throws IOException { - CheckedConsumer indexBuilder = iw -> { - iw.addDocument( - List.of( - new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(1.1)), - new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(2.2)) - ) - ); - iw.addDocument( - List.of( - new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(10.1)), - new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(20.1)) - ) - ); - }; + public void testMultipleDocValuesValues() throws IOException { assertThat( - execute(indexBuilder, "for (double d : doc['foo']) {value(d * 9.9)}", new NumberFieldType("foo", NumberType.DOUBLE)), + multipleValuesInDocValues().collect("for (double d : doc['foo']) {value(d * 9.9)}"), equalTo(List.of(10.89, 21.78, 99.99, 198.99)) ); } + private TestCase randomDoubles() throws IOException { + return testCase(iw -> { + iw.addDocument(List.of(doubleDocValue(randomDouble()))); + iw.addDocument(List.of(doubleDocValue(randomDouble()))); + }); + } + + private TestCase singleValueInSource() throws IOException { + return testCase(iw -> { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 1}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 10}")))); + }); + } + + private TestCase multiValueInSource() throws IOException { + return testCase(iw -> { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [1, 2]}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [10, 20]}")))); + }); + } + + private TestCase singleValueInDocValues() throws IOException { + return testCase(iw -> { + iw.addDocument(List.of(doubleDocValue(1.1))); + iw.addDocument(List.of(doubleDocValue(10.1))); + }); + } + + private TestCase multipleValuesInDocValues() throws IOException { + return testCase(iw -> { + iw.addDocument(List.of(doubleDocValue(1.1), doubleDocValue(2.2))); + iw.addDocument(List.of(doubleDocValue(10.1), doubleDocValue(20.1))); + }); + } + + private IndexableField doubleDocValue(double value) { + return new SortedNumericDocValuesField("foo", NumericUtils.doubleToSortableLong(value)); + } + + @Override + protected MappedFieldType[] fieldTypes() { + return new MappedFieldType[] { new NumberFieldType("foo", NumberType.DOUBLE) }; + } + @Override protected ScriptContext scriptContext() { return DoubleScriptFieldScript.CONTEXT; diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java index 05d48808b5620..6f3aef1fbc589 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java @@ -9,9 +9,8 @@ import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; import org.elasticsearch.script.ScriptContext; @@ -31,57 +30,70 @@ public class LongScriptFieldScriptTests extends ScriptFieldScriptTestCase< Long> { public void testConstant() throws IOException { - CheckedConsumer indexBuilder = iw -> { - iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); - iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); - }; - assertThat(execute(indexBuilder, "value(10)"), equalTo(List.of(10L, 10L))); + assertThat(randomLongs().collect("value(10)"), equalTo(List.of(10L, 10L))); } public void testTwoConstants() throws IOException { - CheckedConsumer indexBuilder = iw -> { + assertThat(randomLongs().collect("value(10); value(20)"), equalTo(List.of(10L, 20L, 10L, 20L))); + } + + public void testSource() throws IOException { + assertThat(singleValueInSource().collect("value(source['foo'] * 10)"), equalTo(List.of(10L, 100L))); + } + + public void testMultileSourceValues() throws IOException { + assertThat(multiValueInSource().collect("for (long l : source['foo']){value(l * 10)}"), equalTo(List.of(10L, 20L, 100L, 200L))); + } + + public void testDocValues() throws IOException { + assertThat(singleValueInDocValues().collect("value(doc['foo'].value * 10)"), equalTo(List.of(10L, 100L))); + } + + public void testMultipleDocValuesValues() throws IOException { + assertThat( + multipleValuesInDocValues().collect("for (long l : doc['foo']) {value(l * 10)}"), + equalTo(List.of(10L, 20L, 100L, 200L)) + ); + } + + private TestCase randomLongs() throws IOException { + return testCase(iw -> { iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); - }; - assertThat(execute(indexBuilder, "value(10); value(20)"), equalTo(List.of(10L, 20L, 10L, 20L))); + }); } - public void testSource() throws IOException { - CheckedConsumer indexBuilder = iw -> { + private TestCase singleValueInSource() throws IOException { + return testCase(iw -> { iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 1}")))); iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 10}")))); - }; - assertThat(execute(indexBuilder, "value(source['foo'] * 10)"), equalTo(List.of(10L, 100L))); + }); } - public void testTwoSourceFields() throws IOException { - CheckedConsumer indexBuilder = iw -> { - iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 1, \"bar\": 2}")))); - iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": 10, \"bar\": 20}")))); - }; - assertThat(execute(indexBuilder, "value(source['foo'] * 10); value(source['bar'] * 10)"), equalTo(List.of(10L, 20L, 100L, 200L))); + private TestCase multiValueInSource() throws IOException { + return testCase(iw -> { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [1, 2]}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [10, 20]}")))); + }); } - public void testDocValues() throws IOException { - CheckedConsumer indexBuilder = iw -> { + private TestCase singleValueInDocValues() throws IOException { + return testCase(iw -> { iw.addDocument(List.of(new SortedNumericDocValuesField("foo", 1))); iw.addDocument(List.of(new SortedNumericDocValuesField("foo", 10))); - }; - assertThat( - execute(indexBuilder, "value(doc['foo'].value * 10)", new NumberFieldType("foo", NumberType.LONG)), - equalTo(List.of(10L, 100L)) - ); + }); } - public void testTwoDocValuesValues() throws IOException { - CheckedConsumer indexBuilder = iw -> { + private TestCase multipleValuesInDocValues() throws IOException { + return testCase(iw -> { iw.addDocument(List.of(new SortedNumericDocValuesField("foo", 1), new SortedNumericDocValuesField("foo", 2))); iw.addDocument(List.of(new SortedNumericDocValuesField("foo", 10), new SortedNumericDocValuesField("foo", 20))); - }; - assertThat( - execute(indexBuilder, "for (long l : doc['foo']) {value(l * 10)}", new NumberFieldType("foo", NumberType.LONG)), - equalTo(List.of(10L, 20L, 100L, 200L)) - ); + }); + } + + @Override + protected MappedFieldType[] fieldTypes() { + return new MappedFieldType[] { new NumberFieldType("foo", NumberType.LONG) }; } @Override diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java index df506d7299eab..6ab7f60347eb1 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java @@ -21,6 +21,7 @@ import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.MappedFieldType; @@ -35,12 +36,12 @@ import org.elasticsearch.search.lookup.DocLookup; import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.ESTestCase; +import org.junit.After; +import java.io.Closeable; import java.io.IOException; -import java.security.AccessController; -import java.security.PrivilegedAction; import java.util.ArrayList; -import java.util.HashMap; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.function.Function; @@ -49,27 +50,10 @@ import static org.mockito.Mockito.when; public abstract class ScriptFieldScriptTestCase extends ESTestCase { - protected abstract ScriptContext scriptContext(); - - protected abstract LF newLeafFactory(F factory, Map params, SourceLookup source, DocLookup fieldData); - - protected abstract S newInstance(LF leafFactory, LeafReaderContext context, List results) throws IOException; - - protected final List execute( - CheckedConsumer indexBuilder, - String script, - MappedFieldType... types - ) throws IOException { - return execute(indexBuilder, script, f -> new MatchAllDocsQuery(), types); - } - - protected final List execute( - CheckedConsumer indexBuilder, - String script, - Function query, - MappedFieldType... types - ) throws IOException { + private final List lazyClose = new ArrayList<>(); + private final ScriptService scriptService; + public ScriptFieldScriptTestCase() { PainlessPlugin painlessPlugin = new PainlessPlugin(); painlessPlugin.loadExtensions(new ExtensionLoader() { @Override @@ -79,50 +63,77 @@ public List loadExtensions(Class extensionPointType) { } }); ScriptModule scriptModule = new ScriptModule(Settings.EMPTY, List.of(painlessPlugin, new RuntimeFields())); - Map params = new HashMap<>(); - SourceLookup source = new SourceLookup(); - MapperService mapperService = mock(MapperService.class); - for (MappedFieldType type : types) { - when(mapperService.fieldType(type.name())).thenReturn(type); - } - Function> fieldDataLookup = ft -> ft.fielddataBuilder("test") - .build(indexSettings(), ft, null, new NoneCircuitBreakerService(), mapperService); - DocLookup fieldData = new DocLookup(mapperService, fieldDataLookup); - try (ScriptService scriptService = new ScriptService(Settings.EMPTY, scriptModule.engines, scriptModule.contexts)) { - F factory = AccessController.doPrivileged( - (PrivilegedAction) () -> scriptService.compile(new Script(script), scriptContext()) - ); - - try (Directory directory = newDirectory(); RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { + scriptService = new ScriptService(Settings.EMPTY, scriptModule.engines, scriptModule.contexts); + } + + protected abstract MappedFieldType[] fieldTypes(); + + protected abstract ScriptContext scriptContext(); + + protected abstract LF newLeafFactory(F factory, Map params, SourceLookup source, DocLookup fieldData); + + protected abstract S newInstance(LF leafFactory, LeafReaderContext context, List results) throws IOException; + + protected final TestCase testCase(CheckedConsumer indexBuilder) throws IOException { + return new TestCase(indexBuilder); + } + + protected class TestCase { + private final SourceLookup sourceLookup = new SourceLookup(); + private final DocLookup fieldData; + private final IndexSearcher searcher; + + private TestCase(CheckedConsumer indexBuilder) throws IOException { + MapperService mapperService = mock(MapperService.class); + for (MappedFieldType type : fieldTypes()) { + when(mapperService.fieldType(type.name())).thenReturn(type); + } + Function> fieldDataLookup = ft -> ft.fielddataBuilder("test") + .build(indexSettings(), ft, null, new NoneCircuitBreakerService(), mapperService); + fieldData = new DocLookup(mapperService, fieldDataLookup); + + Directory directory = newDirectory(); + lazyClose.add(directory); + try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { indexBuilder.accept(indexWriter); - try (DirectoryReader reader = indexWriter.getReader()) { - IndexSearcher searcher = newSearcher(reader); - LF leafFactory = newLeafFactory(factory, params, source, fieldData); - List result = new ArrayList<>(); - searcher.search(query.apply(leafFactory), new Collector() { + DirectoryReader reader = indexWriter.getReader(); + lazyClose.add(reader); + searcher = newSearcher(reader); + } + } + + protected LF script(String script) { + return newLeafFactory(scriptService.compile(new Script(script), scriptContext()), Map.of(), sourceLookup, fieldData); + } + + protected List collect(String script) throws IOException { + return collect(new MatchAllDocsQuery(), script(script)); + } + + protected List collect(Query query, LF script) throws IOException { + List result = new ArrayList<>(); + searcher.search(query, new Collector() { + @Override + public ScoreMode scoreMode() { + return ScoreMode.COMPLETE_NO_SCORES; + } + + @Override + public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { + S compiled = newInstance(script, context, result); + return new LeafCollector() { @Override - public ScoreMode scoreMode() { - return ScoreMode.COMPLETE_NO_SCORES; - } + public void setScorer(Scorable scorer) throws IOException {} @Override - public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { - S compiled = newInstance(leafFactory, context, result); - return new LeafCollector() { - @Override - public void setScorer(Scorable scorer) throws IOException {} - - @Override - public void collect(int doc) throws IOException { - compiled.setDocument(doc); - compiled.execute(); - } - }; + public void collect(int doc) throws IOException { + compiled.setDocument(doc); + compiled.execute(); } - }); - return result; + }; } - } + }); + return result; } } @@ -137,4 +148,10 @@ private IndexSettings indexSettings() { Settings.EMPTY ); } + + @After + public void closeAll() throws IOException { + Collections.reverse(lazyClose); // Close in the oppposite order added so readers close before directory + IOUtils.close(lazyClose); + } } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java index 996782236281b..6a277a53e198a 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java @@ -8,16 +8,17 @@ import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType; +import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.search.lookup.DocLookup; import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; +import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -30,78 +31,52 @@ public class StringScriptFieldScriptTests extends ScriptFieldScriptTestCase< String> { public void testConstant() throws IOException { - CheckedConsumer indexBuilder = iw -> { - iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef(randomAlphaOfLength(2))))); - iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef(randomAlphaOfLength(2))))); - }; - assertThat(execute(indexBuilder, "value('cat')"), equalTo(List.of("cat", "cat"))); + assertThat(randomStrings().collect("value('cat')"), equalTo(List.of("cat", "cat"))); } public void testTwoConstants() throws IOException { - CheckedConsumer indexBuilder = iw -> { - iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef(randomAlphaOfLength(2))))); - iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef(randomAlphaOfLength(2))))); - }; - assertThat(execute(indexBuilder, "value('cat'); value('dog')"), equalTo(List.of("cat", "dog", "cat", "dog"))); + assertThat(randomStrings().collect("value('cat'); value('dog')"), equalTo(List.of("cat", "dog", "cat", "dog"))); } public void testSource() throws IOException { - CheckedConsumer indexBuilder = iw -> { - iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": \"cat\"}")))); - iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": \"dog\"}")))); - }; - assertThat(execute(indexBuilder, "value(source['foo'] + 'o')"), equalTo(List.of("cato", "dogo"))); + assertThat(singleValueInSource().collect("value(source['foo'] + 'o')"), equalTo(List.of("cato", "dogo"))); } - public void testTwoSourceFields() throws IOException { - CheckedConsumer indexBuilder = iw -> { - iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": \"cat\", \"bar\": \"chicken\"}")))); - iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": \"dog\", \"bar\": \"pig\"}")))); - }; + public void testMultipleSourceValues() throws IOException { assertThat( - execute(indexBuilder, "value(source['foo'] + 'o'); value(source['bar'] + 'ie')"), - equalTo(List.of("cato", "chickenie", "dogo", "pigie")) + multiValueInSource().collect("for (String v : source['foo']) {value(v + 'o')}"), + equalTo(List.of("cato", "chickeno", "dogo", "pigo")) ); } public void testDocValues() throws IOException { - CheckedConsumer indexBuilder = iw -> { - iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef("cat")))); - iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef("dog")))); - }; - assertThat(execute(indexBuilder, "value(doc['foo'].value + 'o')", new KeywordFieldType("foo")), equalTo(List.of("cato", "dogo"))); + assertThat(singleValueInDocValues().collect("value(doc['foo'].value + 'o')"), equalTo(List.of("cato", "dogo"))); } - public void testTwoDocValuesValues() throws IOException { - assertThat(execute(farmAnimals(), ADD_O, new KeywordFieldType("foo")), equalTo(List.of("cato", "pigo", "chickeno", "dogo"))); + public void testMultipleDocValuesValues() throws IOException { + assertThat(twoValuesInDocValues().collect(ADD_O), equalTo(List.of("cato", "pigo", "chickeno", "dogo"))); } public void testTermQuery() throws IOException { - assertThat(execute(farmAnimals(), ADD_O, lf -> lf.termQuery("foo", "cat"), new KeywordFieldType("foo")), equalTo(List.of())); - assertThat( - execute(farmAnimals(), ADD_O, lf -> lf.termQuery("foo", "cato"), new KeywordFieldType("foo")), - equalTo(List.of("cato", "pigo")) - ); - assertThat( - execute(farmAnimals(), ADD_O, lf -> lf.termQuery("foo", "dogo"), new KeywordFieldType("foo")), - equalTo(List.of("chickeno", "dogo")) - ); + TestCase c = twoValuesInDocValues(); + StringScriptFieldScript.LeafFactory addO = c.script(ADD_O); + assertThat(c.collect(addO.termQuery("foo", "cat"), addO), equalTo(List.of())); + assertThat(c.collect(addO.termQuery("foo", "cato"), addO), equalTo(List.of("cato", "pigo"))); + assertThat(c.collect(addO.termQuery("foo", "dogo"), addO), equalTo(List.of("chickeno", "dogo"))); } public void testPrefixQuery() throws IOException { - assertThat(execute(farmAnimals(), ADD_O, lf -> lf.prefixQuery("foo", "catdog"), new KeywordFieldType("foo")), equalTo(List.of())); - assertThat( - execute(farmAnimals(), ADD_O, lf -> lf.prefixQuery("foo", "cat"), new KeywordFieldType("foo")), - equalTo(List.of("cato", "pigo")) - ); - assertThat( - execute(farmAnimals(), ADD_O, lf -> lf.prefixQuery("foo", "dogo"), new KeywordFieldType("foo")), - equalTo(List.of("chickeno", "dogo")) - ); - assertThat( - execute(farmAnimals(), ADD_O, lf -> lf.prefixQuery("foo", "c"), new KeywordFieldType("foo")), - equalTo(List.of("cato", "pigo", "chickeno", "dogo")) - ); + TestCase c = twoValuesInDocValues(); + StringScriptFieldScript.LeafFactory addO = c.script(ADD_O); + assertThat(c.collect(addO.prefixQuery("foo", "catdog"), addO), equalTo(List.of())); + assertThat(c.collect(addO.prefixQuery("foo", "cat"), addO), equalTo(List.of("cato", "pigo"))); + assertThat(c.collect(addO.prefixQuery("foo", "dogo"), addO), equalTo(List.of("chickeno", "dogo"))); + assertThat(c.collect(addO.prefixQuery("foo", "d"), addO), equalTo(List.of("chickeno", "dogo"))); + } + + @Override + protected MappedFieldType[] fieldTypes() { + return new MappedFieldType[] { new KeywordFieldType("foo") }; } @Override @@ -128,18 +103,45 @@ protected StringScriptFieldScript newInstance( return leafFactory.newInstance(context, result::add); } - private CheckedConsumer farmAnimals() { - return iw -> { - iw.addDocument( - List.of(new SortedSetDocValuesField("foo", new BytesRef("cat")), new SortedSetDocValuesField("foo", new BytesRef("pig"))) - ); - iw.addDocument( - List.of( - new SortedSetDocValuesField("foo", new BytesRef("chicken")), - new SortedSetDocValuesField("foo", new BytesRef("dog")) - ) - ); - }; + private TestCase randomStrings() throws IOException { + return testCase(iw -> { + iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef(randomAlphaOfLength(2))))); + iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef(randomAlphaOfLength(2))))); + }); + } + + private TestCase singleValueInSource() throws IOException { + return testCase(iw -> { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": \"cat\"}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": \"dog\"}")))); + }); + } + + private TestCase multiValueInSource() throws IOException { + return testCase(iw -> { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [\"cat\", \"chicken\"]}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [\"dog\", \"pig\"]}")))); + }); + } + + private TestCase singleValueInDocValues() throws IOException { + return testCase(iw -> { + iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef("cat")))); + iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef("dog")))); + }); + } + + private TestCase twoValuesInDocValues() throws IOException { + return testCase(iw -> { + List doc = new ArrayList<>(); + doc.add(new SortedSetDocValuesField("foo", new BytesRef("cat"))); + doc.add(new SortedSetDocValuesField("foo", new BytesRef("pig"))); + iw.addDocument(doc); + doc.clear(); + doc.add(new SortedSetDocValuesField("foo", new BytesRef("chicken"))); + doc.add(new SortedSetDocValuesField("foo", new BytesRef("dog"))); + iw.addDocument(doc); + }); } private static final String ADD_O = "for (String s: doc['foo']) {value(s + 'o')}"; From 7d43642db91f41a5a46ec18d79ffd7e2d96e40a6 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 29 Jun 2020 17:05:06 -0400 Subject: [PATCH 09/35] Fails --- .../AbstractScriptFieldScript.java | 3 + .../ScriptFieldScriptTestCase.java | 26 +++- .../StringScriptFieldScriptTests.java | 135 +++++++++++++----- 3 files changed, 126 insertions(+), 38 deletions(-) diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldScript.java index b9d605c796530..b56d89b8b0802 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldScript.java @@ -38,8 +38,11 @@ public AbstractScriptFieldScript(Map params, SourceLookup source public final void setDocument(int docId) { source.setSegmentAndDocument(ctx, docId); fieldData.setDocument(docId); + onSetDocument(docId); } + protected void onSetDocument(int docId) {} + /** * Expose the {@code params} of the script to the script itself. */ diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java index 6ab7f60347eb1..24af059a2eacf 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java @@ -29,10 +29,12 @@ import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; import org.elasticsearch.painless.PainlessPlugin; import org.elasticsearch.plugins.ExtensiblePlugin.ExtensionLoader; +import org.elasticsearch.plugins.ScriptPlugin; import org.elasticsearch.script.Script; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptModule; import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.lookup.DocLookup; import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.test.ESTestCase; @@ -62,10 +64,18 @@ public List loadExtensions(Class extensionPointType) { return (List) List.of(new RuntimeFieldsPainlessExtension()); } }); - ScriptModule scriptModule = new ScriptModule(Settings.EMPTY, List.of(painlessPlugin, new RuntimeFields())); + List scriptPlugins = new ArrayList<>(); + scriptPlugins.add(painlessPlugin); // TODO move painless to integration tests + scriptPlugins.add(new RuntimeFields()); + scriptPlugins.addAll(extraScriptPlugins()); + ScriptModule scriptModule = new ScriptModule(Settings.EMPTY, scriptPlugins); scriptService = new ScriptService(Settings.EMPTY, scriptModule.engines, scriptModule.contexts); } + protected List extraScriptPlugins() { + return List.of(); + } + protected abstract MappedFieldType[] fieldTypes(); protected abstract ScriptContext scriptContext(); @@ -80,18 +90,17 @@ protected final TestCase testCase(CheckedConsumer indexBuilder) throws IOException { - MapperService mapperService = mock(MapperService.class); for (MappedFieldType type : fieldTypes()) { when(mapperService.fieldType(type.name())).thenReturn(type); } Function> fieldDataLookup = ft -> ft.fielddataBuilder("test") .build(indexSettings(), ft, null, new NoneCircuitBreakerService(), mapperService); fieldData = new DocLookup(mapperService, fieldDataLookup); - Directory directory = newDirectory(); lazyClose.add(directory); try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { @@ -103,7 +112,15 @@ private TestCase(CheckedConsumer indexBuilder) t } protected LF script(String script) { - return newLeafFactory(scriptService.compile(new Script(script), scriptContext()), Map.of(), sourceLookup, fieldData); + return script(new Script(script)); + } + + protected LF testScript(String name) { + return script(new Script(ScriptType.INLINE, "test", name, Map.of())); + } + + protected LF script(Script script) { + return newLeafFactory(scriptService.compile(script, scriptContext()), Map.of(), sourceLookup, fieldData); } protected List collect(String script) throws IOException { @@ -111,6 +128,7 @@ protected List collect(String script) throws IOException { } protected List collect(Query query, LF script) throws IOException { + // Now run the query and collect the results List result = new ArrayList<>(); searcher.search(query, new Collector() { @Override diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java index 6a277a53e198a..6f8ea414bc1dd 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java @@ -11,16 +11,22 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType; import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.plugins.ScriptPlugin; import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptEngine; import org.elasticsearch.search.lookup.DocLookup; import org.elasticsearch.search.lookup.SourceLookup; import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; import static org.hamcrest.Matchers.equalTo; @@ -50,7 +56,7 @@ public void testMultipleSourceValues() throws IOException { } public void testDocValues() throws IOException { - assertThat(singleValueInDocValues().collect("value(doc['foo'].value + 'o')"), equalTo(List.of("cato", "dogo"))); + assertThat(singleValueInDocValues().collect(ADD_O), equalTo(List.of("cato", "dogo"))); } public void testMultipleDocValuesValues() throws IOException { @@ -59,50 +65,21 @@ public void testMultipleDocValuesValues() throws IOException { public void testTermQuery() throws IOException { TestCase c = twoValuesInDocValues(); - StringScriptFieldScript.LeafFactory addO = c.script(ADD_O); - assertThat(c.collect(addO.termQuery("foo", "cat"), addO), equalTo(List.of())); + StringScriptFieldScript.LeafFactory addO = c.testScript("add_o"); +// assertThat(c.collect(addO.termQuery("foo", "cat"), addO), equalTo(List.of())); assertThat(c.collect(addO.termQuery("foo", "cato"), addO), equalTo(List.of("cato", "pigo"))); - assertThat(c.collect(addO.termQuery("foo", "dogo"), addO), equalTo(List.of("chickeno", "dogo"))); +// assertThat(c.collect(addO.termQuery("foo", "dogo"), addO), equalTo(List.of("chickeno", "dogo"))); } public void testPrefixQuery() throws IOException { TestCase c = twoValuesInDocValues(); - StringScriptFieldScript.LeafFactory addO = c.script(ADD_O); + StringScriptFieldScript.LeafFactory addO = c.testScript("add_o"); assertThat(c.collect(addO.prefixQuery("foo", "catdog"), addO), equalTo(List.of())); assertThat(c.collect(addO.prefixQuery("foo", "cat"), addO), equalTo(List.of("cato", "pigo"))); assertThat(c.collect(addO.prefixQuery("foo", "dogo"), addO), equalTo(List.of("chickeno", "dogo"))); assertThat(c.collect(addO.prefixQuery("foo", "d"), addO), equalTo(List.of("chickeno", "dogo"))); } - @Override - protected MappedFieldType[] fieldTypes() { - return new MappedFieldType[] { new KeywordFieldType("foo") }; - } - - @Override - protected ScriptContext scriptContext() { - return StringScriptFieldScript.CONTEXT; - } - - @Override - protected StringScriptFieldScript.LeafFactory newLeafFactory( - StringScriptFieldScript.Factory factory, - Map params, - SourceLookup source, - DocLookup fieldData - ) { - return factory.newFactory(params, source, fieldData); - } - - @Override - protected StringScriptFieldScript newInstance( - StringScriptFieldScript.LeafFactory leafFactory, - LeafReaderContext context, - List result - ) throws IOException { - return leafFactory.newInstance(context, result::add); - } - private TestCase randomStrings() throws IOException { return testCase(iw -> { iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef(randomAlphaOfLength(2))))); @@ -145,4 +122,94 @@ private TestCase twoValuesInDocValues() throws IOException { } private static final String ADD_O = "for (String s: doc['foo']) {value(s + 'o')}"; + + @Override + protected List extraScriptPlugins() { + return List.of(new ScriptPlugin() { + @Override + public ScriptEngine getScriptEngine(Settings settings, Collection> contexts) { + return new ScriptEngine() { + @Override + public String getType() { + return "test"; + } + + @Override + public Set> getSupportedContexts() { + return Set.of(StringScriptFieldScript.CONTEXT); + } + + @Override + public FactoryType compile( + String name, + String code, + ScriptContext context, + Map params + ) { + assert context == StringScriptFieldScript.CONTEXT; + @SuppressWarnings("unchecked") + FactoryType result = (FactoryType) compile(name); + return result; + } + + private StringScriptFieldScript.Factory compile(String name) { + if (name.equals("add_o")) { + Set visited = new LinkedHashSet<>(); + return (params, source, fieldData) -> { + StringScriptFieldScript.LeafFactory leafFactory = (ctx, sync) -> { + return new StringScriptFieldScript(params, source, fieldData, ctx, sync) { + @Override + protected void onSetDocument(int docId) { + int rebased = ctx.docBase + docId; + if (false == visited.add(rebased)) { + throw new AssertionError("Visited [" + rebased + "] twice. Order before was " + visited); + } + } + + @Override + public void execute() { + for (Object v : getDoc().get("foo")) { + sync.accept(v + "o"); + } + } + }; + }; + return leafFactory; + }; + } + throw new IllegalArgumentException(); + } + }; + } + }); + } + + @Override + protected MappedFieldType[] fieldTypes() { + return new MappedFieldType[] { new KeywordFieldType("foo") }; + } + + @Override + protected ScriptContext scriptContext() { + return StringScriptFieldScript.CONTEXT; + } + + @Override + protected StringScriptFieldScript.LeafFactory newLeafFactory( + StringScriptFieldScript.Factory factory, + Map params, + SourceLookup source, + DocLookup fieldData + ) { + return factory.newFactory(params, source, fieldData); + } + + @Override + protected StringScriptFieldScript newInstance( + StringScriptFieldScript.LeafFactory leafFactory, + LeafReaderContext context, + List result + ) throws IOException { + return leafFactory.newInstance(context, result::add); + } } From 89d6352d9d3aed37b30487a40ad8b3b0f24060b3 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 30 Jun 2020 10:52:32 -0400 Subject: [PATCH 10/35] Shared iteration --- .../runtimefields/AbstractRuntimeValues.java | 100 ++++++ .../AbstractScriptFieldScript.java | 2 +- .../runtimefields/DoubleRuntimeValues.java | 93 ++++++ .../DoubleScriptFieldScript.java | 10 + .../runtimefields/LongRuntimeValues.java | 121 ++++++++ .../runtimefields/LongScriptFieldScript.java | 10 + .../runtimefields/StringRuntimeValues.java | 285 ++++++++++++++++++ .../StringScriptFieldScript.java | 196 +----------- .../DoubleScriptFieldScriptTests.java | 37 ++- .../LongScriptFieldScriptTests.java | 31 +- .../ScriptFieldScriptTestCase.java | 30 +- .../StringScriptFieldScriptTests.java | 51 ++-- 12 files changed, 716 insertions(+), 250 deletions(-) create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractRuntimeValues.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleRuntimeValues.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongRuntimeValues.java create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractRuntimeValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractRuntimeValues.java new file mode 100644 index 0000000000000..ef5edc1afe4ba --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractRuntimeValues.java @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedNumericDocValues; + +import java.io.IOException; +import java.util.function.IntConsumer; + +/** + * Abstract base for implementing doc values and queries against values + * calculated at runtime. The tricky thing about this is that we'd like to + * calculate the values as few times as possible in case the calculation is + * expensive, but some of the APIs that we rely on to + * calculate the values like {@link SortedNumericDocValues#advanceExact(int)} + * are "forwards only". + *

+ * We solve this in the same way that big cities handle public transportation: + * with a bus! In our case, the bus is subclasses of {@link SharedValues}. + * Queries and doc values are implemented calling {@link #unstarted()} to get + * the {@linkplain SharedValues} that has yet to start iterating. That way + * many queries can share the same underlying {@linkplain SharedValues} + * instance, only calculating the values for a document once. If other code + * needs to iterate the values after the first iteration has started then + * it'll get a new {@linkplain SharedValues} from {@linkplain #unstarted}, + * this "leaving on a different bus". + * + * @param the subtype of {@link SharedValues} needed by the subclass + */ +public abstract class AbstractRuntimeValues.SharedValues> { + private SV unstarted; + + protected final SV unstarted() { + if (unstarted == null) { + unstarted = newSharedValues(); + } + return unstarted; + } + + protected abstract SV newSharedValues(); + + protected abstract class SharedValues { + protected int count; + private boolean sort; + + private int lastDocBase = -1; + private IntConsumer lastLeafCursor; + private int docId = -1; + private int maxDoc; + + protected final IntConsumer leafCursor(LeafReaderContext ctx) throws IOException { + if (lastDocBase != ctx.docBase) { + if (lastDocBase == -1) { + // Now that we're started future iterations can't share these values. + unstarted = null; + } + lastDocBase = ctx.docBase; + IntConsumer leafLoader = newLeafLoader(ctx); + docId = -1; + maxDoc = ctx.reader().maxDoc(); + lastLeafCursor = new IntConsumer() { + @Override + public void accept(int targetDocId) { + if (docId == targetDocId) { + return; + } + docId = targetDocId; + count = 0; + leafLoader.accept(targetDocId); + if (sort) { + sort(); + } + } + }; + } + return lastLeafCursor; + } + + protected final void alwaysSortResults() { + sort = true; + } + + protected final int docId() { + return docId; + } + + protected final int maxDoc() { + return maxDoc; + } + + protected abstract IntConsumer newLeafLoader(LeafReaderContext ctx) throws IOException; + + protected abstract void sort(); + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldScript.java index b56d89b8b0802..2ac6790424889 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldScript.java @@ -35,7 +35,7 @@ public AbstractScriptFieldScript(Map params, SourceLookup source /** * Set the document to run the script against. */ - public final void setDocument(int docId) { + public final void setDocId(int docId) { source.setSegmentAndDocument(ctx, docId); fieldData.setDocument(docId); onSetDocument(docId); diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleRuntimeValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleRuntimeValues.java new file mode 100644 index 0000000000000..7d3c762ed948a --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleRuntimeValues.java @@ -0,0 +1,93 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.util.ArrayUtil; +import org.elasticsearch.common.CheckedFunction; +import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; + +import java.io.IOException; +import java.util.Arrays; +import java.util.function.DoubleConsumer; +import java.util.function.IntConsumer; + +public final class DoubleRuntimeValues extends AbstractRuntimeValues { + @FunctionalInterface + public interface NewLeafLoader { + IntConsumer leafLoader(LeafReaderContext ctx, DoubleConsumer sync) throws IOException; + } + + private final NewLeafLoader newLeafLoader; + + public DoubleRuntimeValues(NewLeafLoader newLeafLoader) { + this.newLeafLoader = newLeafLoader; + } + + public CheckedFunction docValues() { + return unstarted().docValues(); + } + + @Override + protected SharedValues newSharedValues() { + return new SharedValues(); + } + + protected class SharedValues extends AbstractRuntimeValues.SharedValues { + private double[] values = new double[1]; + + @Override + protected IntConsumer newLeafLoader(LeafReaderContext ctx) throws IOException { + return newLeafLoader.leafLoader(ctx, this::add); + } + + private void add(double value) { + int newCount = count + 1; + if (values.length < newCount) { + values = Arrays.copyOf(values, ArrayUtil.oversize(newCount, 8)); + } + values[count] = value; + count = newCount; + } + + @Override + protected void sort() { + Arrays.sort(values, 0, count); + } + + private CheckedFunction docValues() { + alwaysSortResults(); + return DocValues::new; + } + + private class DocValues extends SortedNumericDoubleValues { + private final IntConsumer leafCursor; + private int next; + + DocValues(LeafReaderContext ctx) throws IOException { + leafCursor = leafCursor(ctx); + } + + @Override + public double nextValue() throws IOException { + return values[next++]; + } + + @Override + public int docValueCount() { + return count; + } + + @Override + public boolean advanceExact(int target) throws IOException { + leafCursor.accept(target); + next = 0; + return count > 0; + } + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java index e86bfb20f71b6..363ddec4a5248 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java @@ -34,6 +34,16 @@ public interface Factory extends ScriptFactory { public interface LeafFactory { DoubleScriptFieldScript newInstance(LeafReaderContext ctx, DoubleConsumer sync) throws IOException; + + default DoubleRuntimeValues runtimeValues() throws IOException { + return new DoubleRuntimeValues((ctx, sync) -> { + DoubleScriptFieldScript script = newInstance(ctx, sync); + return docId -> { + script.setDocId(docId); + script.execute(); + }; + }); + } } private final DoubleConsumer sync; diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongRuntimeValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongRuntimeValues.java new file mode 100644 index 0000000000000..104ae76b78be8 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongRuntimeValues.java @@ -0,0 +1,121 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.util.ArrayUtil; +import org.elasticsearch.common.CheckedFunction; + +import java.io.IOException; +import java.util.Arrays; +import java.util.function.IntConsumer; +import java.util.function.LongConsumer; + +public final class LongRuntimeValues extends AbstractRuntimeValues { + @FunctionalInterface + public interface NewLeafLoader { + IntConsumer leafLoader(LeafReaderContext ctx, LongConsumer sync) throws IOException; + } + + private final NewLeafLoader newLeafLoader; + + public LongRuntimeValues(NewLeafLoader newLeafLoader) { + this.newLeafLoader = newLeafLoader; + } + + public CheckedFunction docValues() { + return unstarted().docValues(); + } + + @Override + protected SharedValues newSharedValues() { + return new SharedValues(); + } + + protected class SharedValues extends AbstractRuntimeValues.SharedValues { + private long[] values = new long[1]; + + @Override + protected IntConsumer newLeafLoader(LeafReaderContext ctx) throws IOException { + return newLeafLoader.leafLoader(ctx, this::add); + } + + private void add(long value) { + int newCount = count + 1; + if (values.length < newCount) { + values = Arrays.copyOf(values, ArrayUtil.oversize(newCount, 8)); + } + values[count] = value; + count = newCount; + } + + @Override + protected void sort() { + Arrays.sort(values, 0, count); + } + + private CheckedFunction docValues() { + alwaysSortResults(); + return DocValues::new; + } + + private class DocValues extends SortedNumericDocValues { + private final IntConsumer leafCursor; + private int next; + + DocValues(LeafReaderContext ctx) throws IOException { + leafCursor = leafCursor(ctx); + } + + @Override + public long nextValue() throws IOException { + return values[next++]; + } + + @Override + public int docValueCount() { + return count; + } + + @Override + public boolean advanceExact(int target) throws IOException { + leafCursor.accept(target); + next = 0; + return count > 0; + } + + @Override + public int docID() { + return docId(); + } + + @Override + public int nextDoc() throws IOException { + return advance(docId() + 1); + } + + @Override + public int advance(int target) throws IOException { + int current = target; + while (current < maxDoc()) { + if (advanceExact(current)) { + return current; + } + current++; + } + return NO_MORE_DOCS; + } + + @Override + public long cost() { + // TODO we have no idea what this should be and no real way to get one + return 1000; + } + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java index e99fa4a944ba3..003d7ce78f15d 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java @@ -34,6 +34,16 @@ public interface Factory extends ScriptFactory { public interface LeafFactory { LongScriptFieldScript newInstance(LeafReaderContext ctx, LongConsumer sync) throws IOException; + + default LongRuntimeValues runtimeValues() throws IOException { + return new LongRuntimeValues((ctx, sync) -> { + LongScriptFieldScript script = newInstance(ctx, sync); + return docId -> { + script.setDocId(docId); + script.execute(); + }; + }); + } } private final LongConsumer sync; diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java new file mode 100644 index 0000000000000..809cf5defc630 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java @@ -0,0 +1,285 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.TwoPhaseIterator; +import org.apache.lucene.search.Weight; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefBuilder; +import org.apache.lucene.util.RamUsageEstimator; +import org.apache.lucene.util.automaton.ByteRunAutomaton; +import org.elasticsearch.common.CheckedFunction; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Objects; +import java.util.Set; +import java.util.function.Consumer; +import java.util.function.IntConsumer; + +public final class StringRuntimeValues extends AbstractRuntimeValues { + @FunctionalInterface + public interface NewLeafLoader { + IntConsumer leafLoader(LeafReaderContext ctx, Consumer sync) throws IOException; + } + + private final NewLeafLoader newLeafLoader; + + public StringRuntimeValues(NewLeafLoader newLeafLoader) { + this.newLeafLoader = newLeafLoader; + } + + public CheckedFunction docValues() { + return unstarted().docValues(); + } + + public Query termQuery(String fieldName, String value) { + return unstarted().new TermQuery(fieldName, value); + } + + public Query prefixQuery(String fieldName, String value) { + return unstarted().new PrefixQuery(fieldName, value); + } + + @Override + protected SharedValues newSharedValues() { + return new SharedValues(); + } + + protected class SharedValues extends AbstractRuntimeValues.SharedValues { + private String[] values = new String[1]; + + @Override + protected IntConsumer newLeafLoader(LeafReaderContext ctx) throws IOException { + return newLeafLoader.leafLoader(ctx, this::add); + } + + private void add(String value) { + if (value == null) { + return; + } + int newCount = count + 1; + if (values.length < newCount) { + values = Arrays.copyOf(values, ArrayUtil.oversize(newCount, RamUsageEstimator.NUM_BYTES_OBJECT_REF)); + } + values[count] = value; + count = newCount; + } + + @Override + protected void sort() { + Arrays.sort(values, 0, count); + } + + private CheckedFunction docValues() { + alwaysSortResults(); + return DocValues::new; + } + + private class DocValues extends SortedBinaryDocValues { + private final BytesRefBuilder ref = new BytesRefBuilder(); + private final IntConsumer leafCursor; + private int next; + + DocValues(LeafReaderContext ctx) throws IOException { + leafCursor = leafCursor(ctx); + } + + @Override + public boolean advanceExact(int docId) throws IOException { + leafCursor.accept(docId); + next = 0; + return count > 0; + } + + @Override + public int docValueCount() { + return count; + } + + @Override + public BytesRef nextValue() throws IOException { + ref.copyChars(values[next++]); + return ref.get(); + } + } + + private class TermQuery extends Query { + private final String fieldName; + private final String term; + + private TermQuery(String fieldName, String term) { + this.fieldName = fieldName; + this.term = term; + } + + @Override + public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { + return new ConstantScoreWeight(this, boost) { + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return false; // scripts aren't really cacheable at this point + } + + @Override + public Scorer scorer(LeafReaderContext ctx) throws IOException { + IntConsumer leafCursor = leafCursor(ctx); + DocIdSetIterator approximation = DocIdSetIterator.all(ctx.reader().maxDoc()); + TwoPhaseIterator twoPhase = new TwoPhaseIterator(approximation) { + @Override + public boolean matches() throws IOException { + leafCursor.accept(approximation.docID()); + for (int i = 0; i < count; i++) { + if (term.equals(values[i])) { + return true; + } + } + return false; + } + + @Override + public float matchCost() { + // TODO we have no idea what this should be and no real way to get one + return 1000f; + } + }; + return new ConstantScoreScorer(this, score(), scoreMode, twoPhase); + } + + @Override + public void extractTerms(Set terms) { + terms.add(new Term(fieldName, term)); + } + }; + } + + @Override + public void visit(QueryVisitor visitor) { + visitor.consumeTerms(this, new Term(fieldName, term)); + } + + @Override + public String toString(String field) { + if (fieldName.contentEquals(field)) { + return term; + } + return fieldName + ":" + term; + } + + @Override + public int hashCode() { + return Objects.hash(fieldName, term); + } + + @Override + public boolean equals(Object obj) { + if (obj == null || getClass() != obj.getClass()) { + return false; + } + TermQuery other = (TermQuery) obj; + return fieldName.equals(other.fieldName) && term.equals(other.term); + } + } + + private class PrefixQuery extends Query { + private final String fieldName; + private final String prefix; + + private PrefixQuery(String fieldName, String prefix) { + this.fieldName = fieldName; + this.prefix = prefix; + } + + @Override + public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { + return new ConstantScoreWeight(this, boost) { + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return false; // scripts aren't really cacheable at this point + } + + @Override + public Scorer scorer(LeafReaderContext ctx) throws IOException { + IntConsumer leafCursor = leafCursor(ctx); + DocIdSetIterator approximation = DocIdSetIterator.all(ctx.reader().maxDoc()); + TwoPhaseIterator twoPhase = new TwoPhaseIterator(approximation) { + @Override + public boolean matches() throws IOException { + leafCursor.accept(approximation.docID()); + for (int i = 0; i < count; i++) { + if (values[i].startsWith(prefix)) { + return true; + } + } + return false; + } + + @Override + public float matchCost() { + // TODO we have no idea what this should be and no real way to get one + return 1000f; + } + }; + return new ConstantScoreScorer(this, score(), scoreMode, twoPhase); + } + + @Override + public void extractTerms(Set terms) { + // TODO doing this is sort of difficult and maybe not needed. + } + }; + } + + @Override + public void visit(QueryVisitor visitor) { + visitor.consumeTermsMatching( + this, + fieldName, + () -> new ByteRunAutomaton( + org.apache.lucene.search.PrefixQuery.toAutomaton(new BytesRef(prefix)), + true, + Integer.MAX_VALUE + ) + ); + } + + @Override + public String toString(String field) { + if (fieldName.contentEquals(field)) { + return prefix + "*"; + } + return fieldName + ":" + prefix + "*"; + } + + @Override + public int hashCode() { + return Objects.hash(fieldName, prefix); + } + + @Override + public boolean equals(Object obj) { + if (obj == null || getClass() != obj.getClass()) { + return false; + } + PrefixQuery other = (PrefixQuery) obj; + return fieldName.equals(other.fieldName) && prefix.equals(other.prefix); + } + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java index 19a1edd451039..801864cb3eef2 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java @@ -7,20 +7,6 @@ package org.elasticsearch.xpack.runtimefields; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.ConstantScoreScorer; -import org.apache.lucene.search.ConstantScoreWeight; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.PrefixQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.QueryVisitor; -import org.apache.lucene.search.ScoreMode; -import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.TwoPhaseIterator; -import org.apache.lucene.search.Weight; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.automaton.ByteRunAutomaton; import org.elasticsearch.painless.spi.Whitelist; import org.elasticsearch.painless.spi.WhitelistLoader; import org.elasticsearch.script.ScriptContext; @@ -31,8 +17,6 @@ import java.io.IOException; import java.util.List; import java.util.Map; -import java.util.Objects; -import java.util.Set; import java.util.function.Consumer; public abstract class StringScriptFieldScript extends AbstractScriptFieldScript { @@ -51,12 +35,14 @@ public interface Factory extends ScriptFactory { public interface LeafFactory { StringScriptFieldScript newInstance(LeafReaderContext ctx, Consumer sync) throws IOException; - default Query termQuery(String fieldName, String value) { - return new ScriptTermQuery(this, fieldName, value); - } - - default Query prefixQuery(String fieldName, String value) { - return new ScriptPrefixQuery(this, fieldName, value); + default StringRuntimeValues runtimeValues() throws IOException { + return new StringRuntimeValues((ctx, sync) -> { + StringScriptFieldScript script = newInstance(ctx, sync); + return docId -> { + script.setDocId(docId); + script.execute(); + }; + }); } } @@ -84,170 +70,4 @@ public void value(String v) { script.sync.accept(v); } } - - private static class ScriptTermQuery extends Query { - private final LeafFactory leafFactory; - private final String fieldName; - private final String term; - - private ScriptTermQuery(LeafFactory leafFactory, String fieldName, String term) { - this.leafFactory = leafFactory; - this.fieldName = fieldName; - this.term = term; - } - - @Override - public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { - return new ConstantScoreWeight(this, boost) { - @Override - public boolean isCacheable(LeafReaderContext ctx) { - return false; // scripts aren't really cacheable at this point - } - - @Override - public Scorer scorer(LeafReaderContext context) throws IOException { - DocIdSetIterator approximation = DocIdSetIterator.all(context.reader().maxDoc()); - TwoPhaseIterator twoPhase = new TwoPhaseIterator(approximation) { - private boolean hasMatches; - private final StringScriptFieldScript script = leafFactory.newInstance( - context, - s -> { hasMatches = hasMatches || term.equals(s); } - ); - - @Override - public boolean matches() throws IOException { - hasMatches = false; - script.setDocument(approximation.docID()); - script.execute(); - return hasMatches; - } - - @Override - public float matchCost() { - // TODO we have no idea what this should be and no real way to get one - return 1000f; - } - }; - return new ConstantScoreScorer(this, score(), scoreMode, twoPhase); - } - - @Override - public void extractTerms(Set terms) { - terms.add(new Term(fieldName, term)); - } - }; - } - - @Override - public void visit(QueryVisitor visitor) { - visitor.consumeTerms(this, new Term(fieldName, term)); - } - - @Override - public String toString(String field) { - if (fieldName.contentEquals(field)) { - return term; - } - return fieldName + ":" + term; - } - - @Override - public int hashCode() { - return Objects.hash(fieldName, term); - } - - @Override - public boolean equals(Object obj) { - if (obj == null || getClass() != obj.getClass()) { - return false; - } - ScriptTermQuery other = (ScriptTermQuery) obj; - return fieldName.equals(other.fieldName) && term.equals(other.term); - } - } - - private static class ScriptPrefixQuery extends Query { - private final LeafFactory leafFactory; - private final String fieldName; - private final String prefix; - - private ScriptPrefixQuery(LeafFactory leafFactory, String fieldName, String prefix) { - this.leafFactory = leafFactory; - this.fieldName = fieldName; - this.prefix = prefix; - } - - @Override - public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { - return new ConstantScoreWeight(this, boost) { - @Override - public boolean isCacheable(LeafReaderContext ctx) { - return false; // scripts aren't really cacheable at this point - } - - @Override - public Scorer scorer(LeafReaderContext context) throws IOException { - DocIdSetIterator approximation = DocIdSetIterator.all(context.reader().maxDoc()); - TwoPhaseIterator twoPhase = new TwoPhaseIterator(approximation) { - private boolean hasMatches; - private final StringScriptFieldScript script = leafFactory.newInstance( - context, - s -> { hasMatches = hasMatches || s.startsWith(prefix); } - ); - - @Override - public boolean matches() throws IOException { - hasMatches = false; - script.setDocument(approximation.docID()); - script.execute(); - return hasMatches; - } - - @Override - public float matchCost() { - // TODO we have no idea what this should be and no real way to get one - return 1000f; - } - }; - return new ConstantScoreScorer(this, score(), scoreMode, twoPhase); - } - - @Override - public void extractTerms(Set terms) { - // TODO doing this is sort of difficult and maybe not needed. - } - }; - } - - @Override - public void visit(QueryVisitor visitor) { - visitor.consumeTermsMatching( - this, - fieldName, - () -> new ByteRunAutomaton(PrefixQuery.toAutomaton(new BytesRef(prefix)), true, Integer.MAX_VALUE) - ); - } - - @Override - public String toString(String field) { - if (fieldName.contentEquals(field)) { - return prefix + "*"; - } - return fieldName + ":" + prefix + "*"; - } - - @Override - public int hashCode() { - return Objects.hash(fieldName, prefix); - } - - @Override - public boolean equals(Object obj) { - if (obj == null || getClass() != obj.getClass()) { - return false; - } - ScriptPrefixQuery other = (ScriptPrefixQuery) obj; - return fieldName.equals(other.fieldName) && prefix.equals(other.prefix); - } - } } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java index e8446010908fe..3cea6203d0a6a 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java @@ -12,23 +12,27 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; +import org.elasticsearch.common.CheckedFunction; +import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.search.lookup.DocLookup; import org.elasticsearch.search.lookup.SourceLookup; +import org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript.Factory; import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.function.Consumer; import static org.hamcrest.Matchers.equalTo; public class DoubleScriptFieldScriptTests extends ScriptFieldScriptTestCase< - DoubleScriptFieldScript, DoubleScriptFieldScript.Factory, - DoubleScriptFieldScript.LeafFactory, + DoubleRuntimeValues, + SortedNumericDoubleValues, Double> { public void testConstant() throws IOException { @@ -36,7 +40,7 @@ public void testConstant() throws IOException { } public void testTwoConstants() throws IOException { - assertThat(randomDoubles().collect("value(3.14); value(2.72)"), equalTo(List.of(3.14, 2.72, 3.14, 2.72))); + assertThat(randomDoubles().collect("value(3.14); value(2.72)"), equalTo(List.of(2.72, 3.14, 2.72, 3.14))); } public void testSource() throws IOException { @@ -111,21 +115,22 @@ protected ScriptContext scriptContext() { } @Override - protected DoubleScriptFieldScript.LeafFactory newLeafFactory( - DoubleScriptFieldScript.Factory factory, - Map params, - SourceLookup source, - DocLookup fieldData - ) { - return factory.newFactory(params, source, fieldData); + protected DoubleRuntimeValues newValues(Factory factory, Map params, SourceLookup source, DocLookup fieldData) + throws IOException { + return factory.newFactory(params, source, fieldData).runtimeValues(); } @Override - protected DoubleScriptFieldScript newInstance( - DoubleScriptFieldScript.LeafFactory leafFactory, - LeafReaderContext context, - List result - ) throws IOException { - return leafFactory.newInstance(context, result::add); + protected CheckedFunction docValuesBuilder(DoubleRuntimeValues values) { + return values.docValues(); + } + + @Override + protected void readAllDocValues(SortedNumericDoubleValues docValues, int docId, Consumer sync) throws IOException { + assertTrue(docValues.advanceExact(docId)); + int count = docValues.docValueCount(); + for (int i = 0; i < count; i++) { + sync.accept(docValues.nextValue()); + } } } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java index 6f3aef1fbc589..668ec75c95039 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java @@ -9,24 +9,28 @@ import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.search.lookup.DocLookup; import org.elasticsearch.search.lookup.SourceLookup; +import org.elasticsearch.xpack.runtimefields.LongScriptFieldScript.Factory; import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.function.Consumer; import static org.hamcrest.Matchers.equalTo; public class LongScriptFieldScriptTests extends ScriptFieldScriptTestCase< - LongScriptFieldScript, LongScriptFieldScript.Factory, - LongScriptFieldScript.LeafFactory, + LongRuntimeValues, + SortedNumericDocValues, Long> { public void testConstant() throws IOException { @@ -102,19 +106,22 @@ protected ScriptContext scriptContext() { } @Override - protected LongScriptFieldScript.LeafFactory newLeafFactory( - LongScriptFieldScript.Factory factory, - Map params, - SourceLookup source, - DocLookup fieldData - ) { - return factory.newFactory(params, source, fieldData); + protected LongRuntimeValues newValues(Factory factory, Map params, SourceLookup source, DocLookup fieldData) + throws IOException { + return factory.newFactory(params, source, fieldData).runtimeValues(); } @Override - protected LongScriptFieldScript newInstance(LongScriptFieldScript.LeafFactory leafFactory, LeafReaderContext context, List result) - throws IOException { + protected CheckedFunction docValuesBuilder(LongRuntimeValues values) { + return values.docValues(); + } - return leafFactory.newInstance(context, result::add); + @Override + protected void readAllDocValues(SortedNumericDocValues docValues, int docId, Consumer sync) throws IOException { + assertTrue(docValues.advanceExact(docId)); + int count = docValues.docValueCount(); + for (int i = 0; i < count; i++) { + sync.accept(docValues.nextValue()); + } } } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java index 24af059a2eacf..a985b480a0632 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java @@ -20,6 +20,7 @@ import org.elasticsearch.Version; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.index.IndexSettings; @@ -46,12 +47,13 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Consumer; import java.util.function.Function; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -public abstract class ScriptFieldScriptTestCase extends ESTestCase { +public abstract class ScriptFieldScriptTestCase extends ESTestCase { private final List lazyClose = new ArrayList<>(); private final ScriptService scriptService; @@ -80,9 +82,11 @@ protected List extraScriptPlugins() { protected abstract ScriptContext scriptContext(); - protected abstract LF newLeafFactory(F factory, Map params, SourceLookup source, DocLookup fieldData); + protected abstract V newValues(F factory, Map params, SourceLookup source, DocLookup fieldData) throws IOException; - protected abstract S newInstance(LF leafFactory, LeafReaderContext context, List results) throws IOException; + protected abstract CheckedFunction docValuesBuilder(V values); + + protected abstract void readAllDocValues(DV docValues, int docId, Consumer sync) throws IOException; protected final TestCase testCase(CheckedConsumer indexBuilder) throws IOException { return new TestCase(indexBuilder); @@ -111,25 +115,26 @@ private TestCase(CheckedConsumer indexBuilder) t } } - protected LF script(String script) { + protected V script(String script) throws IOException { return script(new Script(script)); } - protected LF testScript(String name) { + protected V testScript(String name) throws IOException { return script(new Script(ScriptType.INLINE, "test", name, Map.of())); } - protected LF script(Script script) { - return newLeafFactory(scriptService.compile(script, scriptContext()), Map.of(), sourceLookup, fieldData); + protected V script(Script script) throws IOException { + return newValues(scriptService.compile(script, scriptContext()), Map.of(), sourceLookup, fieldData); } protected List collect(String script) throws IOException { return collect(new MatchAllDocsQuery(), script(script)); } - protected List collect(Query query, LF script) throws IOException { + protected List collect(Query query, V values) throws IOException { // Now run the query and collect the results List result = new ArrayList<>(); + CheckedFunction docValuesBuilder = docValuesBuilder(values); searcher.search(query, new Collector() { @Override public ScoreMode scoreMode() { @@ -137,16 +142,15 @@ public ScoreMode scoreMode() { } @Override - public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { - S compiled = newInstance(script, context, result); + public LeafCollector getLeafCollector(LeafReaderContext ctx) throws IOException { + DV docValues = docValuesBuilder.apply(ctx); return new LeafCollector() { @Override public void setScorer(Scorable scorer) throws IOException {} @Override - public void collect(int doc) throws IOException { - compiled.setDocument(doc); - compiled.execute(); + public void collect(int docId) throws IOException { + readAllDocValues(docValues, docId, result::add); } }; } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java index 6f8ea414bc1dd..7607436758c51 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java @@ -11,14 +11,17 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.plugins.ScriptPlugin; import org.elasticsearch.script.ScriptContext; import org.elasticsearch.script.ScriptEngine; import org.elasticsearch.search.lookup.DocLookup; import org.elasticsearch.search.lookup.SourceLookup; +import org.elasticsearch.xpack.runtimefields.StringScriptFieldScript.Factory; import java.io.IOException; import java.util.ArrayList; @@ -27,15 +30,18 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.function.Consumer; import static org.hamcrest.Matchers.equalTo; public class StringScriptFieldScriptTests extends ScriptFieldScriptTestCase< - StringScriptFieldScript, StringScriptFieldScript.Factory, - StringScriptFieldScript.LeafFactory, + StringRuntimeValues, + SortedBinaryDocValues, String> { + private final Set visited = new LinkedHashSet<>(); + public void testConstant() throws IOException { assertThat(randomStrings().collect("value('cat')"), equalTo(List.of("cat", "cat"))); } @@ -65,18 +71,23 @@ public void testMultipleDocValuesValues() throws IOException { public void testTermQuery() throws IOException { TestCase c = twoValuesInDocValues(); - StringScriptFieldScript.LeafFactory addO = c.testScript("add_o"); -// assertThat(c.collect(addO.termQuery("foo", "cat"), addO), equalTo(List.of())); + StringRuntimeValues addO = c.testScript("add_o"); + assertThat(c.collect(addO.termQuery("foo", "cat"), addO), equalTo(List.of())); + visited.clear(); assertThat(c.collect(addO.termQuery("foo", "cato"), addO), equalTo(List.of("cato", "pigo"))); -// assertThat(c.collect(addO.termQuery("foo", "dogo"), addO), equalTo(List.of("chickeno", "dogo"))); + visited.clear(); + assertThat(c.collect(addO.termQuery("foo", "dogo"), addO), equalTo(List.of("chickeno", "dogo"))); } public void testPrefixQuery() throws IOException { TestCase c = twoValuesInDocValues(); - StringScriptFieldScript.LeafFactory addO = c.testScript("add_o"); + StringRuntimeValues addO = c.testScript("add_o"); assertThat(c.collect(addO.prefixQuery("foo", "catdog"), addO), equalTo(List.of())); + visited.clear(); assertThat(c.collect(addO.prefixQuery("foo", "cat"), addO), equalTo(List.of("cato", "pigo"))); + visited.clear(); assertThat(c.collect(addO.prefixQuery("foo", "dogo"), addO), equalTo(List.of("chickeno", "dogo"))); + visited.clear(); assertThat(c.collect(addO.prefixQuery("foo", "d"), addO), equalTo(List.of("chickeno", "dogo"))); } @@ -154,7 +165,6 @@ public FactoryType compile( private StringScriptFieldScript.Factory compile(String name) { if (name.equals("add_o")) { - Set visited = new LinkedHashSet<>(); return (params, source, fieldData) -> { StringScriptFieldScript.LeafFactory leafFactory = (ctx, sync) -> { return new StringScriptFieldScript(params, source, fieldData, ctx, sync) { @@ -195,21 +205,22 @@ protected ScriptContext scriptContext() { } @Override - protected StringScriptFieldScript.LeafFactory newLeafFactory( - StringScriptFieldScript.Factory factory, - Map params, - SourceLookup source, - DocLookup fieldData - ) { - return factory.newFactory(params, source, fieldData); + protected StringRuntimeValues newValues(Factory factory, Map params, SourceLookup source, DocLookup fieldData) + throws IOException { + return factory.newFactory(params, source, fieldData).runtimeValues(); + } + + @Override + protected CheckedFunction docValuesBuilder(StringRuntimeValues values) { + return values.docValues(); } @Override - protected StringScriptFieldScript newInstance( - StringScriptFieldScript.LeafFactory leafFactory, - LeafReaderContext context, - List result - ) throws IOException { - return leafFactory.newInstance(context, result::add); + protected void readAllDocValues(SortedBinaryDocValues docValues, int docId, Consumer sync) throws IOException { + assertTrue(docValues.advanceExact(docId)); + int count = docValues.docValueCount(); + for (int i = 0; i < count; i++) { + sync.accept(docValues.nextValue().utf8ToString()); + } } } From 5b353ce0b2866ac95947eed8c8145d1edd03a58f Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 30 Jun 2020 11:02:39 -0400 Subject: [PATCH 11/35] Long term query --- .../runtimefields/LongRuntimeValues.java | 94 +++++++++++++++++++ .../LongScriptFieldScriptTests.java | 81 ++++++++++++++++ .../StringScriptFieldScriptTests.java | 12 ++- 3 files changed, 183 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongRuntimeValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongRuntimeValues.java index 104ae76b78be8..69696bee67b7f 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongRuntimeValues.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongRuntimeValues.java @@ -8,11 +8,24 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.TwoPhaseIterator; +import org.apache.lucene.search.Weight; import org.apache.lucene.util.ArrayUtil; import org.elasticsearch.common.CheckedFunction; import java.io.IOException; import java.util.Arrays; +import java.util.Objects; +import java.util.Set; import java.util.function.IntConsumer; import java.util.function.LongConsumer; @@ -32,6 +45,10 @@ public CheckedFunction d return unstarted().docValues(); } + public Query termQuery(String fieldName, long value) { + return unstarted().new TermQuery(fieldName, value); + } + @Override protected SharedValues newSharedValues() { return new SharedValues(); @@ -117,5 +134,82 @@ public long cost() { return 1000; } } + + private class TermQuery extends Query { + private final String fieldName; + private final long term; + + private TermQuery(String fieldName, long term) { + this.fieldName = fieldName; + this.term = term; + } + + @Override + public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { + return new ConstantScoreWeight(this, boost) { + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return false; // scripts aren't really cacheable at this point + } + + @Override + public Scorer scorer(LeafReaderContext ctx) throws IOException { + IntConsumer leafCursor = leafCursor(ctx); + DocIdSetIterator approximation = DocIdSetIterator.all(ctx.reader().maxDoc()); + TwoPhaseIterator twoPhase = new TwoPhaseIterator(approximation) { + @Override + public boolean matches() throws IOException { + leafCursor.accept(approximation.docID()); + for (int i = 0; i < count; i++) { + if (term == values[i]) { + return true; + } + } + return false; + } + + @Override + public float matchCost() { + // TODO we have no idea what this should be and no real way to get one + return 1000f; + } + }; + return new ConstantScoreScorer(this, score(), scoreMode, twoPhase); + } + + @Override + public void extractTerms(Set terms) { + terms.add(new Term(fieldName, Long.toString(term))); + } + }; + } + + @Override + public void visit(QueryVisitor visitor) { + visitor.consumeTerms(this, new Term(fieldName, Long.toString(term))); + } + + @Override + public String toString(String field) { + if (fieldName.contentEquals(field)) { + return Long.toString(term); + } + return fieldName + ":" + term; + } + + @Override + public int hashCode() { + return Objects.hash(fieldName, term); + } + + @Override + public boolean equals(Object obj) { + if (obj == null || getClass() != obj.getClass()) { + return false; + } + TermQuery other = (TermQuery) obj; + return fieldName.equals(other.fieldName) && term == other.term; + } + } } } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java index 668ec75c95039..447103db86125 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java @@ -12,17 +12,23 @@ import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.CheckedFunction; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; +import org.elasticsearch.plugins.ScriptPlugin; import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptEngine; import org.elasticsearch.search.lookup.DocLookup; import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.xpack.runtimefields.LongScriptFieldScript.Factory; import java.io.IOException; +import java.util.Collection; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Consumer; import static org.hamcrest.Matchers.equalTo; @@ -33,6 +39,8 @@ public class LongScriptFieldScriptTests extends ScriptFieldScriptTestCase< SortedNumericDocValues, Long> { + private final Set visited = new LinkedHashSet<>(); + public void testConstant() throws IOException { assertThat(randomLongs().collect("value(10)"), equalTo(List.of(10L, 10L))); } @@ -60,6 +68,19 @@ public void testMultipleDocValuesValues() throws IOException { ); } + public void testTermQuery() throws IOException { + TestCase c = multipleValuesInDocValues(); + LongRuntimeValues addO = c.testScript("times_ten"); + assertThat(c.collect(addO.termQuery("foo", 1), addO), equalTo(List.of())); + visited.clear(); + assertThat(c.collect(addO.termQuery("foo", 10), addO), equalTo(List.of(10L, 20L))); + visited.clear(); + assertThat(c.collect(addO.termQuery("foo", 20), addO), equalTo(List.of(10L, 20L))); + visited.clear(); + assertThat(c.collect(addO.termQuery("foo", 100), addO), equalTo(List.of(100L, 200L))); + } + + private TestCase randomLongs() throws IOException { return testCase(iw -> { iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); @@ -124,4 +145,64 @@ protected void readAllDocValues(SortedNumericDocValues docValues, int docId, Con sync.accept(docValues.nextValue()); } } + + @Override + protected List extraScriptPlugins() { + return List.of(new ScriptPlugin() { + @Override + public ScriptEngine getScriptEngine(Settings settings, Collection> contexts) { + return new ScriptEngine() { + @Override + public String getType() { + return "test"; + } + + @Override + public Set> getSupportedContexts() { + return Set.of(LongScriptFieldScript.CONTEXT); + } + + @Override + public FactoryType compile( + String name, + String code, + ScriptContext context, + Map params + ) { + assert context == LongScriptFieldScript.CONTEXT; + @SuppressWarnings("unchecked") + FactoryType result = (FactoryType) compile(name); + return result; + } + + private LongScriptFieldScript.Factory compile(String name) { + if (name.equals("times_ten")) { + return (params, source, fieldData) -> { + LongScriptFieldScript.LeafFactory leafFactory = (ctx, sync) -> { + return new LongScriptFieldScript(params, source, fieldData, ctx, sync) { + @Override + protected void onSetDocument(int docId) { + int rebased = ctx.docBase + docId; + if (false == visited.add(rebased)) { + throw new AssertionError("Visited [" + rebased + "] twice. Order before was " + visited); + } + } + + @Override + public void execute() { + for (Object v : getDoc().get("foo")) { + sync.accept(((long) v) * 10); + } + } + }; + }; + return leafFactory; + }; + } + throw new IllegalArgumentException(); + } + }; + } + }); + } } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java index 7607436758c51..1b8008f35980d 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java @@ -66,26 +66,30 @@ public void testDocValues() throws IOException { } public void testMultipleDocValuesValues() throws IOException { - assertThat(twoValuesInDocValues().collect(ADD_O), equalTo(List.of("cato", "pigo", "chickeno", "dogo"))); + assertThat(multipleValuesInDocValues().collect(ADD_O), equalTo(List.of("cato", "pigo", "chickeno", "dogo"))); } public void testTermQuery() throws IOException { - TestCase c = twoValuesInDocValues(); + TestCase c = multipleValuesInDocValues(); StringRuntimeValues addO = c.testScript("add_o"); assertThat(c.collect(addO.termQuery("foo", "cat"), addO), equalTo(List.of())); visited.clear(); assertThat(c.collect(addO.termQuery("foo", "cato"), addO), equalTo(List.of("cato", "pigo"))); visited.clear(); + assertThat(c.collect(addO.termQuery("foo", "pigo"), addO), equalTo(List.of("cato", "pigo"))); + visited.clear(); assertThat(c.collect(addO.termQuery("foo", "dogo"), addO), equalTo(List.of("chickeno", "dogo"))); } public void testPrefixQuery() throws IOException { - TestCase c = twoValuesInDocValues(); + TestCase c = multipleValuesInDocValues(); StringRuntimeValues addO = c.testScript("add_o"); assertThat(c.collect(addO.prefixQuery("foo", "catdog"), addO), equalTo(List.of())); visited.clear(); assertThat(c.collect(addO.prefixQuery("foo", "cat"), addO), equalTo(List.of("cato", "pigo"))); visited.clear(); + assertThat(c.collect(addO.prefixQuery("foo", "pig"), addO), equalTo(List.of("cato", "pigo"))); + visited.clear(); assertThat(c.collect(addO.prefixQuery("foo", "dogo"), addO), equalTo(List.of("chickeno", "dogo"))); visited.clear(); assertThat(c.collect(addO.prefixQuery("foo", "d"), addO), equalTo(List.of("chickeno", "dogo"))); @@ -119,7 +123,7 @@ private TestCase singleValueInDocValues() throws IOException { }); } - private TestCase twoValuesInDocValues() throws IOException { + private TestCase multipleValuesInDocValues() throws IOException { return testCase(iw -> { List doc = new ArrayList<>(); doc.add(new SortedSetDocValuesField("foo", new BytesRef("cat"))); From 23af46631e832988e0e28f32458899fa6c9a33c4 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 30 Jun 2020 11:34:43 -0400 Subject: [PATCH 12/35] Common stuffs? --- .../runtimefields/AbstractRuntimeValues.java | 75 ++++++++++ .../runtimefields/LongRuntimeValues.java | 113 +++++++-------- .../runtimefields/StringRuntimeValues.java | 133 ++++-------------- .../LongScriptFieldScriptTests.java | 13 ++ 4 files changed, 173 insertions(+), 161 deletions(-) diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractRuntimeValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractRuntimeValues.java index ef5edc1afe4ba..0b0ea2d67bf00 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractRuntimeValues.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractRuntimeValues.java @@ -8,8 +8,18 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.TwoPhaseIterator; +import org.apache.lucene.search.Weight; import java.io.IOException; +import java.util.Objects; import java.util.function.IntConsumer; /** @@ -96,5 +106,70 @@ protected final int maxDoc() { protected abstract IntConsumer newLeafLoader(LeafReaderContext ctx) throws IOException; protected abstract void sort(); + + public abstract class AbstractRuntimeQuery extends Query { + protected final String fieldName; + + protected AbstractRuntimeQuery(String fieldName) { + this.fieldName = fieldName; + } + + @Override + public final Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { + return new ConstantScoreWeight(this, boost) { + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return false; // scripts aren't really cacheable at this point + } + + @Override + public Scorer scorer(LeafReaderContext ctx) throws IOException { + IntConsumer leafCursor = leafCursor(ctx); + DocIdSetIterator approximation = DocIdSetIterator.all(ctx.reader().maxDoc()); + TwoPhaseIterator twoPhase = new TwoPhaseIterator(approximation) { + @Override + public boolean matches() throws IOException { + leafCursor.accept(approximation.docID()); + return AbstractRuntimeQuery.this.matches(); + } + + @Override + public float matchCost() { + // TODO we have no idea what this should be and no real way to get one + return 1000f; + } + }; + return new ConstantScoreScorer(this, score(), scoreMode, twoPhase); + } + }; + } + + protected abstract boolean matches(); + + @Override + public final String toString(String field) { + if (fieldName.contentEquals(field)) { + return bareToString(); + } + return fieldName + ":" + bareToString(); + } + + protected abstract String bareToString(); + + @Override + public int hashCode() { + return Objects.hash(fieldName); + } + + @Override + public boolean equals(Object obj) { + if (obj == null || getClass() != obj.getClass()) { + return false; + } + @SuppressWarnings("unchecked") + AbstractRuntimeQuery other = (AbstractRuntimeQuery) obj; + return fieldName.equals(other.fieldName); + } + } } } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongRuntimeValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongRuntimeValues.java index 69696bee67b7f..9eb571a5c2967 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongRuntimeValues.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongRuntimeValues.java @@ -9,23 +9,14 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.Term; -import org.apache.lucene.search.ConstantScoreScorer; -import org.apache.lucene.search.ConstantScoreWeight; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryVisitor; -import org.apache.lucene.search.ScoreMode; -import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.TwoPhaseIterator; -import org.apache.lucene.search.Weight; import org.apache.lucene.util.ArrayUtil; import org.elasticsearch.common.CheckedFunction; import java.io.IOException; import java.util.Arrays; import java.util.Objects; -import java.util.Set; import java.util.function.IntConsumer; import java.util.function.LongConsumer; @@ -49,6 +40,10 @@ public Query termQuery(String fieldName, long value) { return unstarted().new TermQuery(fieldName, value); } + public Query rangeQuery(String fieldName, long lowerValue, long upperValue) { + return unstarted().new RangeQuery(fieldName, lowerValue, upperValue); + } + @Override protected SharedValues newSharedValues() { return new SharedValues(); @@ -135,53 +130,22 @@ public long cost() { } } - private class TermQuery extends Query { - private final String fieldName; + private class TermQuery extends AbstractRuntimeQuery { private final long term; private TermQuery(String fieldName, long term) { - this.fieldName = fieldName; + super(fieldName); this.term = term; } @Override - public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { - return new ConstantScoreWeight(this, boost) { - @Override - public boolean isCacheable(LeafReaderContext ctx) { - return false; // scripts aren't really cacheable at this point - } - - @Override - public Scorer scorer(LeafReaderContext ctx) throws IOException { - IntConsumer leafCursor = leafCursor(ctx); - DocIdSetIterator approximation = DocIdSetIterator.all(ctx.reader().maxDoc()); - TwoPhaseIterator twoPhase = new TwoPhaseIterator(approximation) { - @Override - public boolean matches() throws IOException { - leafCursor.accept(approximation.docID()); - for (int i = 0; i < count; i++) { - if (term == values[i]) { - return true; - } - } - return false; - } - - @Override - public float matchCost() { - // TODO we have no idea what this should be and no real way to get one - return 1000f; - } - }; - return new ConstantScoreScorer(this, score(), scoreMode, twoPhase); + protected boolean matches() { + for (int i = 0; i < count; i++) { + if (term == values[i]) { + return true; } - - @Override - public void extractTerms(Set terms) { - terms.add(new Term(fieldName, Long.toString(term))); - } - }; + } + return false; } @Override @@ -190,25 +154,62 @@ public void visit(QueryVisitor visitor) { } @Override - public String toString(String field) { - if (fieldName.contentEquals(field)) { - return Long.toString(term); - } - return fieldName + ":" + term; + protected String bareToString() { + return Long.toString(term); } @Override public int hashCode() { - return Objects.hash(fieldName, term); + return Objects.hash(super.hashCode(), term); } @Override public boolean equals(Object obj) { - if (obj == null || getClass() != obj.getClass()) { + if (false == super.equals(obj)) { return false; } TermQuery other = (TermQuery) obj; - return fieldName.equals(other.fieldName) && term == other.term; + return term == other.term; + } + } + + private class RangeQuery extends AbstractRuntimeQuery { + private final long lowerValue; + private final long upperValue; + + private RangeQuery(String fieldName, long lowerValue, long upperValue) { + super(fieldName); + this.lowerValue = lowerValue; + this.upperValue = upperValue; + } + + @Override + protected boolean matches() { + for (int i = 0; i < count; i++) { + if (lowerValue <= values[i] && values[i] <= upperValue) { + return true; + } + } + return false; + } + + @Override + protected String bareToString() { + return "[" + lowerValue + "," + upperValue + "]"; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), lowerValue, upperValue); + } + + @Override + public boolean equals(Object obj) { + if (false == super.equals(obj)) { + return false; + } + RangeQuery other = (RangeQuery) obj; + return lowerValue == other.lowerValue && upperValue == other.upperValue; } } } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java index 809cf5defc630..ce68d7bf51b48 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java @@ -8,16 +8,8 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.Term; -import org.apache.lucene.search.ConstantScoreScorer; -import org.apache.lucene.search.ConstantScoreWeight; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryVisitor; -import org.apache.lucene.search.ScoreMode; -import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.TwoPhaseIterator; -import org.apache.lucene.search.Weight; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; @@ -29,7 +21,6 @@ import java.io.IOException; import java.util.Arrays; import java.util.Objects; -import java.util.Set; import java.util.function.Consumer; import java.util.function.IntConsumer; @@ -120,53 +111,22 @@ public BytesRef nextValue() throws IOException { } } - private class TermQuery extends Query { - private final String fieldName; + private class TermQuery extends AbstractRuntimeQuery { private final String term; private TermQuery(String fieldName, String term) { - this.fieldName = fieldName; + super(fieldName); this.term = term; } @Override - public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { - return new ConstantScoreWeight(this, boost) { - @Override - public boolean isCacheable(LeafReaderContext ctx) { - return false; // scripts aren't really cacheable at this point + protected boolean matches() { + for (int i = 0; i < count; i++) { + if (term.equals(values[i])) { + return true; } - - @Override - public Scorer scorer(LeafReaderContext ctx) throws IOException { - IntConsumer leafCursor = leafCursor(ctx); - DocIdSetIterator approximation = DocIdSetIterator.all(ctx.reader().maxDoc()); - TwoPhaseIterator twoPhase = new TwoPhaseIterator(approximation) { - @Override - public boolean matches() throws IOException { - leafCursor.accept(approximation.docID()); - for (int i = 0; i < count; i++) { - if (term.equals(values[i])) { - return true; - } - } - return false; - } - - @Override - public float matchCost() { - // TODO we have no idea what this should be and no real way to get one - return 1000f; - } - }; - return new ConstantScoreScorer(this, score(), scoreMode, twoPhase); - } - - @Override - public void extractTerms(Set terms) { - terms.add(new Term(fieldName, term)); - } - }; + } + return false; } @Override @@ -175,75 +135,46 @@ public void visit(QueryVisitor visitor) { } @Override - public String toString(String field) { - if (fieldName.contentEquals(field)) { - return term; - } - return fieldName + ":" + term; + protected String bareToString() { + return term; } @Override public int hashCode() { - return Objects.hash(fieldName, term); + return Objects.hash(super.hashCode(), term); } @Override public boolean equals(Object obj) { - if (obj == null || getClass() != obj.getClass()) { + if (false == super.equals(obj)) { return false; } TermQuery other = (TermQuery) obj; - return fieldName.equals(other.fieldName) && term.equals(other.term); + return term.equals(other.term); } } - private class PrefixQuery extends Query { - private final String fieldName; + private class PrefixQuery extends AbstractRuntimeQuery { private final String prefix; private PrefixQuery(String fieldName, String prefix) { - this.fieldName = fieldName; + super(fieldName); this.prefix = prefix; } @Override - public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { - return new ConstantScoreWeight(this, boost) { - @Override - public boolean isCacheable(LeafReaderContext ctx) { - return false; // scripts aren't really cacheable at this point - } - - @Override - public Scorer scorer(LeafReaderContext ctx) throws IOException { - IntConsumer leafCursor = leafCursor(ctx); - DocIdSetIterator approximation = DocIdSetIterator.all(ctx.reader().maxDoc()); - TwoPhaseIterator twoPhase = new TwoPhaseIterator(approximation) { - @Override - public boolean matches() throws IOException { - leafCursor.accept(approximation.docID()); - for (int i = 0; i < count; i++) { - if (values[i].startsWith(prefix)) { - return true; - } - } - return false; - } - - @Override - public float matchCost() { - // TODO we have no idea what this should be and no real way to get one - return 1000f; - } - }; - return new ConstantScoreScorer(this, score(), scoreMode, twoPhase); + protected boolean matches() { + for (int i = 0; i < count; i++) { + if (values[i].startsWith(prefix)) { + return true; } + } + return false; + } - @Override - public void extractTerms(Set terms) { - // TODO doing this is sort of difficult and maybe not needed. - } - }; + @Override + protected String bareToString() { + return prefix + "*"; } @Override @@ -259,26 +190,18 @@ public void visit(QueryVisitor visitor) { ); } - @Override - public String toString(String field) { - if (fieldName.contentEquals(field)) { - return prefix + "*"; - } - return fieldName + ":" + prefix + "*"; - } - @Override public int hashCode() { - return Objects.hash(fieldName, prefix); + return Objects.hash(super.hashCode(), prefix); } @Override public boolean equals(Object obj) { - if (obj == null || getClass() != obj.getClass()) { + if (false == super.equals(obj)) { return false; } PrefixQuery other = (PrefixQuery) obj; - return fieldName.equals(other.fieldName) && prefix.equals(other.prefix); + return prefix.equals(other.prefix); } } } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java index 447103db86125..cefa16f66dba6 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java @@ -80,6 +80,19 @@ public void testTermQuery() throws IOException { assertThat(c.collect(addO.termQuery("foo", 100), addO), equalTo(List.of(100L, 200L))); } + public void testRangeQuery() throws IOException { + TestCase c = multipleValuesInDocValues(); + LongRuntimeValues addO = c.testScript("times_ten"); + assertThat(c.collect(addO.rangeQuery("foo", 1, 2), addO), equalTo(List.of())); + visited.clear(); + assertThat(c.collect(addO.rangeQuery("foo", 9, 11), addO), equalTo(List.of(10L, 20L))); + visited.clear(); + assertThat(c.collect(addO.rangeQuery("foo", 10, 11), addO), equalTo(List.of(10L, 20L))); + visited.clear(); + assertThat(c.collect(addO.rangeQuery("foo", 19, 21), addO), equalTo(List.of(10L, 20L))); + visited.clear(); + assertThat(c.collect(addO.rangeQuery("foo", 99, 101), addO), equalTo(List.of(100L, 200L))); + } private TestCase randomLongs() throws IOException { return testCase(iw -> { From acf64c8b3f9840548a6648480f461318f138ffd2 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 30 Jun 2020 11:59:59 -0400 Subject: [PATCH 13/35] Double queries --- .../runtimefields/DoubleRuntimeValues.java | 145 ++++++++++++++++++ .../runtimefields/LongRuntimeValues.java | 50 ++++++ .../DoubleScriptFieldScriptTests.java | 110 +++++++++++++ .../LongScriptFieldScriptTests.java | 14 ++ 4 files changed, 319 insertions(+) diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleRuntimeValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleRuntimeValues.java index 7d3c762ed948a..9842fa8b69cc5 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleRuntimeValues.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleRuntimeValues.java @@ -7,12 +7,16 @@ package org.elasticsearch.xpack.runtimefields; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; import org.apache.lucene.util.ArrayUtil; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import java.io.IOException; import java.util.Arrays; +import java.util.Objects; import java.util.function.DoubleConsumer; import java.util.function.IntConsumer; @@ -32,6 +36,18 @@ public CheckedFunction 0; } } + + private class TermQuery extends AbstractRuntimeQuery { + private final double term; + + private TermQuery(String fieldName, double term) { + super(fieldName); + this.term = term; + } + + @Override + protected boolean matches() { + for (int i = 0; i < count; i++) { + if (term == values[i]) { + return true; + } + } + return false; + } + + @Override + public void visit(QueryVisitor visitor) { + visitor.consumeTerms(this, new Term(fieldName, Double.toString(term))); + } + + @Override + protected String bareToString() { + return Double.toString(term); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), term); + } + + @Override + public boolean equals(Object obj) { + if (false == super.equals(obj)) { + return false; + } + TermQuery other = (TermQuery) obj; + return term == other.term; + } + } + + private class TermsQuery extends AbstractRuntimeQuery { + private final double[] terms; + + private TermsQuery(String fieldName, double[] terms) { + super(fieldName); + this.terms = terms.clone(); + Arrays.sort(terms); + } + + @Override + protected boolean matches() { + for (int i = 0; i < count; i++) { + if (Arrays.binarySearch(terms, values[i]) >= 0) { + return true; + } + } + return false; + } + + @Override + public void visit(QueryVisitor visitor) { + for (double term : terms) { + visitor.consumeTerms(this, new Term(fieldName, Double.toString(term))); + } + } + + @Override + protected String bareToString() { + return "{" + Arrays.toString(terms) + "}"; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), Arrays.hashCode(terms)); + } + + @Override + public boolean equals(Object obj) { + if (false == super.equals(obj)) { + return false; + } + TermsQuery other = (TermsQuery) obj; + return Arrays.equals(terms, other.terms); + } + } + + private class RangeQuery extends AbstractRuntimeQuery { + private final double lowerValue; + private final double upperValue; + + private RangeQuery(String fieldName, double lowerValue, double upperValue) { + super(fieldName); + this.lowerValue = lowerValue; + this.upperValue = upperValue; + } + + @Override + protected boolean matches() { + for (int i = 0; i < count; i++) { + if (lowerValue <= values[i] && values[i] <= upperValue) { + return true; + } + } + return false; + } + + @Override + protected String bareToString() { + return "[" + lowerValue + "," + upperValue + "]"; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), lowerValue, upperValue); + } + + @Override + public boolean equals(Object obj) { + if (false == super.equals(obj)) { + return false; + } + RangeQuery other = (RangeQuery) obj; + return lowerValue == other.lowerValue && upperValue == other.upperValue; + } + } } } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongRuntimeValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongRuntimeValues.java index 9eb571a5c2967..e1ae4ea3ee8a1 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongRuntimeValues.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongRuntimeValues.java @@ -40,6 +40,10 @@ public Query termQuery(String fieldName, long value) { return unstarted().new TermQuery(fieldName, value); } + public Query termsQuery(String fieldName, long... value) { + return unstarted().new TermsQuery(fieldName, value); + } + public Query rangeQuery(String fieldName, long lowerValue, long upperValue) { return unstarted().new RangeQuery(fieldName, lowerValue, upperValue); } @@ -173,6 +177,52 @@ public boolean equals(Object obj) { } } + private class TermsQuery extends AbstractRuntimeQuery { + private final long[] terms; + + private TermsQuery(String fieldName, long[] terms) { + super(fieldName); + this.terms = terms.clone(); + Arrays.sort(terms); + } + + @Override + protected boolean matches() { + for (int i = 0; i < count; i++) { + if (Arrays.binarySearch(terms, values[i]) >= 0) { + return true; + } + } + return false; + } + + @Override + public void visit(QueryVisitor visitor) { + for (long term : terms) { + visitor.consumeTerms(this, new Term(fieldName, Long.toString(term))); + } + } + + @Override + protected String bareToString() { + return "{" + Arrays.toString(terms) + "}"; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), Arrays.hashCode(terms)); + } + + @Override + public boolean equals(Object obj) { + if (false == super.equals(obj)) { + return false; + } + TermsQuery other = (TermsQuery) obj; + return Arrays.equals(terms, other.terms); + } + } + private class RangeQuery extends AbstractRuntimeQuery { private final long lowerValue; private final long upperValue; diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java index 3cea6203d0a6a..11c8970b8c152 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java @@ -13,18 +13,24 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.NumericUtils; import org.elasticsearch.common.CheckedFunction; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; +import org.elasticsearch.plugins.ScriptPlugin; import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptEngine; import org.elasticsearch.search.lookup.DocLookup; import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript.Factory; import java.io.IOException; +import java.util.Collection; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Consumer; import static org.hamcrest.Matchers.equalTo; @@ -35,6 +41,8 @@ public class DoubleScriptFieldScriptTests extends ScriptFieldScriptTestCase< SortedNumericDoubleValues, Double> { + private final Set visited = new LinkedHashSet<>(); + public void testConstant() throws IOException { assertThat(randomDoubles().collect("value(3.14)"), equalTo(List.of(3.14, 3.14))); } @@ -65,6 +73,48 @@ public void testMultipleDocValuesValues() throws IOException { ); } + public void testTermQuery() throws IOException { + TestCase c = multipleValuesInDocValues(); + DoubleRuntimeValues addO = c.testScript("times_nine_point_nine"); + assertThat(c.collect(addO.termQuery("foo", 1), addO), equalTo(List.of())); + visited.clear(); + assertThat(c.collect(addO.termQuery("foo", 10.89), addO), equalTo(List.of(10.89, 21.78))); + visited.clear(); + assertThat(c.collect(addO.termQuery("foo", 21.78), addO), equalTo(List.of(10.89, 21.78))); + visited.clear(); + assertThat(c.collect(addO.termQuery("foo", 99.99), addO), equalTo(List.of(99.99, 198.99))); + } + + public void testTermsQuery() throws IOException { + TestCase c = multipleValuesInDocValues(); + DoubleRuntimeValues addO = c.testScript("times_nine_point_nine"); + assertThat(c.collect(addO.termsQuery("foo", 1, 2), addO), equalTo(List.of())); + visited.clear(); + assertThat(c.collect(addO.termsQuery("foo", 10.89, 11), addO), equalTo(List.of(10.89, 21.78))); + visited.clear(); + assertThat(c.collect(addO.termsQuery("foo", 21.78, 22), addO), equalTo(List.of(10.89, 21.78))); + visited.clear(); + assertThat(c.collect(addO.termsQuery("foo", 20, 21.78), addO), equalTo(List.of(10.89, 21.78))); + visited.clear(); + assertThat(c.collect(addO.termsQuery("foo", 99.99, 100), addO), equalTo(List.of(99.99, 198.99))); + } + + public void testRangeQuery() throws IOException { + TestCase c = multipleValuesInDocValues(); + DoubleRuntimeValues addO = c.testScript("times_nine_point_nine"); + assertThat(c.collect(addO.rangeQuery("foo", 1, 2), addO), equalTo(List.of())); + visited.clear(); + assertThat(c.collect(addO.rangeQuery("foo", 9, 11), addO), equalTo(List.of(10.89, 21.78))); + visited.clear(); + assertThat(c.collect(addO.rangeQuery("foo", 10.89, 11), addO), equalTo(List.of(10.89, 21.78))); + visited.clear(); + assertThat(c.collect(addO.rangeQuery("foo", 21.78, 22), addO), equalTo(List.of(10.89, 21.78))); + visited.clear(); + assertThat(c.collect(addO.rangeQuery("foo", 21, 21.78), addO), equalTo(List.of(10.89, 21.78))); + visited.clear(); + assertThat(c.collect(addO.rangeQuery("foo", 99, 100), addO), equalTo(List.of(99.99, 198.99))); + } + private TestCase randomDoubles() throws IOException { return testCase(iw -> { iw.addDocument(List.of(doubleDocValue(randomDouble()))); @@ -133,4 +183,64 @@ protected void readAllDocValues(SortedNumericDoubleValues docValues, int docId, sync.accept(docValues.nextValue()); } } + + @Override + protected List extraScriptPlugins() { + return List.of(new ScriptPlugin() { + @Override + public ScriptEngine getScriptEngine(Settings settings, Collection> contexts) { + return new ScriptEngine() { + @Override + public String getType() { + return "test"; + } + + @Override + public Set> getSupportedContexts() { + return Set.of(DoubleScriptFieldScript.CONTEXT); + } + + @Override + public FactoryType compile( + String name, + String code, + ScriptContext context, + Map params + ) { + assert context == DoubleScriptFieldScript.CONTEXT; + @SuppressWarnings("unchecked") + FactoryType result = (FactoryType) compile(name); + return result; + } + + private DoubleScriptFieldScript.Factory compile(String name) { + if (name.equals("times_nine_point_nine")) { + return (params, source, fieldData) -> { + DoubleScriptFieldScript.LeafFactory leafFactory = (ctx, sync) -> { + return new DoubleScriptFieldScript(params, source, fieldData, ctx, sync) { + @Override + protected void onSetDocument(int docId) { + int rebased = ctx.docBase + docId; + if (false == visited.add(rebased)) { + throw new AssertionError("Visited [" + rebased + "] twice. Order before was " + visited); + } + } + + @Override + public void execute() { + for (Object v : getDoc().get("foo")) { + sync.accept(((double) v) * 9.9); + } + } + }; + }; + return leafFactory; + }; + } + throw new IllegalArgumentException(); + } + }; + } + }); + } } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java index cefa16f66dba6..dfafe13d56ba0 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java @@ -80,6 +80,20 @@ public void testTermQuery() throws IOException { assertThat(c.collect(addO.termQuery("foo", 100), addO), equalTo(List.of(100L, 200L))); } + public void testTermsQuery() throws IOException { + TestCase c = multipleValuesInDocValues(); + LongRuntimeValues addO = c.testScript("times_ten"); + assertThat(c.collect(addO.termsQuery("foo", 1, 2), addO), equalTo(List.of())); + visited.clear(); + assertThat(c.collect(addO.termsQuery("foo", 10, 11), addO), equalTo(List.of(10L, 20L))); + visited.clear(); + assertThat(c.collect(addO.termsQuery("foo", 20, 21), addO), equalTo(List.of(10L, 20L))); + visited.clear(); + assertThat(c.collect(addO.termsQuery("foo", 19, 20), addO), equalTo(List.of(10L, 20L))); + visited.clear(); + assertThat(c.collect(addO.termsQuery("foo", 100, 11), addO), equalTo(List.of(100L, 200L))); + } + public void testRangeQuery() throws IOException { TestCase c = multipleValuesInDocValues(); LongRuntimeValues addO = c.testScript("times_ten"); From 7ee42c0feb697307d08afff1c63b1ed2205bd5dd Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 30 Jun 2020 12:52:58 -0400 Subject: [PATCH 14/35] Rename --- .../DoubleScriptFieldScriptTests.java | 36 ++++++++--------- .../LongScriptFieldScriptTests.java | 40 +++++++++---------- .../ScriptFieldScriptTestCase.java | 6 ++- 3 files changed, 42 insertions(+), 40 deletions(-) diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java index 11c8970b8c152..abd02fb90f242 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java @@ -75,44 +75,44 @@ public void testMultipleDocValuesValues() throws IOException { public void testTermQuery() throws IOException { TestCase c = multipleValuesInDocValues(); - DoubleRuntimeValues addO = c.testScript("times_nine_point_nine"); - assertThat(c.collect(addO.termQuery("foo", 1), addO), equalTo(List.of())); + DoubleRuntimeValues timesTen = c.testScript("times_nine_point_nine"); + assertThat(c.collect(timesTen.termQuery("foo", 1), timesTen), equalTo(List.of())); visited.clear(); - assertThat(c.collect(addO.termQuery("foo", 10.89), addO), equalTo(List.of(10.89, 21.78))); + assertThat(c.collect(timesTen.termQuery("foo", 10.89), timesTen), equalTo(List.of(10.89, 21.78))); visited.clear(); - assertThat(c.collect(addO.termQuery("foo", 21.78), addO), equalTo(List.of(10.89, 21.78))); + assertThat(c.collect(timesTen.termQuery("foo", 21.78), timesTen), equalTo(List.of(10.89, 21.78))); visited.clear(); - assertThat(c.collect(addO.termQuery("foo", 99.99), addO), equalTo(List.of(99.99, 198.99))); + assertThat(c.collect(timesTen.termQuery("foo", 99.99), timesTen), equalTo(List.of(99.99, 198.99))); } public void testTermsQuery() throws IOException { TestCase c = multipleValuesInDocValues(); - DoubleRuntimeValues addO = c.testScript("times_nine_point_nine"); - assertThat(c.collect(addO.termsQuery("foo", 1, 2), addO), equalTo(List.of())); + DoubleRuntimeValues timesTen = c.testScript("times_nine_point_nine"); + assertThat(c.collect(timesTen.termsQuery("foo", 1, 2), timesTen), equalTo(List.of())); visited.clear(); - assertThat(c.collect(addO.termsQuery("foo", 10.89, 11), addO), equalTo(List.of(10.89, 21.78))); + assertThat(c.collect(timesTen.termsQuery("foo", 10.89, 11), timesTen), equalTo(List.of(10.89, 21.78))); visited.clear(); - assertThat(c.collect(addO.termsQuery("foo", 21.78, 22), addO), equalTo(List.of(10.89, 21.78))); + assertThat(c.collect(timesTen.termsQuery("foo", 21.78, 22), timesTen), equalTo(List.of(10.89, 21.78))); visited.clear(); - assertThat(c.collect(addO.termsQuery("foo", 20, 21.78), addO), equalTo(List.of(10.89, 21.78))); + assertThat(c.collect(timesTen.termsQuery("foo", 20, 21.78), timesTen), equalTo(List.of(10.89, 21.78))); visited.clear(); - assertThat(c.collect(addO.termsQuery("foo", 99.99, 100), addO), equalTo(List.of(99.99, 198.99))); + assertThat(c.collect(timesTen.termsQuery("foo", 99.99, 100), timesTen), equalTo(List.of(99.99, 198.99))); } public void testRangeQuery() throws IOException { TestCase c = multipleValuesInDocValues(); - DoubleRuntimeValues addO = c.testScript("times_nine_point_nine"); - assertThat(c.collect(addO.rangeQuery("foo", 1, 2), addO), equalTo(List.of())); + DoubleRuntimeValues timesTen = c.testScript("times_nine_point_nine"); + assertThat(c.collect(timesTen.rangeQuery("foo", 1, 2), timesTen), equalTo(List.of())); visited.clear(); - assertThat(c.collect(addO.rangeQuery("foo", 9, 11), addO), equalTo(List.of(10.89, 21.78))); + assertThat(c.collect(timesTen.rangeQuery("foo", 9, 11), timesTen), equalTo(List.of(10.89, 21.78))); visited.clear(); - assertThat(c.collect(addO.rangeQuery("foo", 10.89, 11), addO), equalTo(List.of(10.89, 21.78))); + assertThat(c.collect(timesTen.rangeQuery("foo", 10.89, 11), timesTen), equalTo(List.of(10.89, 21.78))); visited.clear(); - assertThat(c.collect(addO.rangeQuery("foo", 21.78, 22), addO), equalTo(List.of(10.89, 21.78))); + assertThat(c.collect(timesTen.rangeQuery("foo", 21.78, 22), timesTen), equalTo(List.of(10.89, 21.78))); visited.clear(); - assertThat(c.collect(addO.rangeQuery("foo", 21, 21.78), addO), equalTo(List.of(10.89, 21.78))); + assertThat(c.collect(timesTen.rangeQuery("foo", 21, 21.78), timesTen), equalTo(List.of(10.89, 21.78))); visited.clear(); - assertThat(c.collect(addO.rangeQuery("foo", 99, 100), addO), equalTo(List.of(99.99, 198.99))); + assertThat(c.collect(timesTen.rangeQuery("foo", 99, 100), timesTen), equalTo(List.of(99.99, 198.99))); } private TestCase randomDoubles() throws IOException { diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java index dfafe13d56ba0..6c71fb0f4bd9a 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java @@ -62,50 +62,48 @@ public void testDocValues() throws IOException { } public void testMultipleDocValuesValues() throws IOException { - assertThat( - multipleValuesInDocValues().collect("for (long l : doc['foo']) {value(l * 10)}"), - equalTo(List.of(10L, 20L, 100L, 200L)) - ); + TestCase c = multipleValuesInDocValues(); + assertThat(multipleValuesInDocValues().collect(c.testScript("times_ten")), equalTo(List.of(10L, 20L, 100L, 200L))); } public void testTermQuery() throws IOException { TestCase c = multipleValuesInDocValues(); - LongRuntimeValues addO = c.testScript("times_ten"); - assertThat(c.collect(addO.termQuery("foo", 1), addO), equalTo(List.of())); + LongRuntimeValues timesTen = c.testScript("times_ten"); + assertThat(c.collect(timesTen.termQuery("foo", 1), timesTen), equalTo(List.of())); visited.clear(); - assertThat(c.collect(addO.termQuery("foo", 10), addO), equalTo(List.of(10L, 20L))); + assertThat(c.collect(timesTen.termQuery("foo", 10), timesTen), equalTo(List.of(10L, 20L))); visited.clear(); - assertThat(c.collect(addO.termQuery("foo", 20), addO), equalTo(List.of(10L, 20L))); + assertThat(c.collect(timesTen.termQuery("foo", 20), timesTen), equalTo(List.of(10L, 20L))); visited.clear(); - assertThat(c.collect(addO.termQuery("foo", 100), addO), equalTo(List.of(100L, 200L))); + assertThat(c.collect(timesTen.termQuery("foo", 100), timesTen), equalTo(List.of(100L, 200L))); } public void testTermsQuery() throws IOException { TestCase c = multipleValuesInDocValues(); - LongRuntimeValues addO = c.testScript("times_ten"); - assertThat(c.collect(addO.termsQuery("foo", 1, 2), addO), equalTo(List.of())); + LongRuntimeValues timesTen = c.testScript("times_ten"); + assertThat(c.collect(timesTen.termsQuery("foo", 1, 2), timesTen), equalTo(List.of())); visited.clear(); - assertThat(c.collect(addO.termsQuery("foo", 10, 11), addO), equalTo(List.of(10L, 20L))); + assertThat(c.collect(timesTen.termsQuery("foo", 10, 11), timesTen), equalTo(List.of(10L, 20L))); visited.clear(); - assertThat(c.collect(addO.termsQuery("foo", 20, 21), addO), equalTo(List.of(10L, 20L))); + assertThat(c.collect(timesTen.termsQuery("foo", 20, 21), timesTen), equalTo(List.of(10L, 20L))); visited.clear(); - assertThat(c.collect(addO.termsQuery("foo", 19, 20), addO), equalTo(List.of(10L, 20L))); + assertThat(c.collect(timesTen.termsQuery("foo", 19, 20), timesTen), equalTo(List.of(10L, 20L))); visited.clear(); - assertThat(c.collect(addO.termsQuery("foo", 100, 11), addO), equalTo(List.of(100L, 200L))); + assertThat(c.collect(timesTen.termsQuery("foo", 100, 11), timesTen), equalTo(List.of(100L, 200L))); } public void testRangeQuery() throws IOException { TestCase c = multipleValuesInDocValues(); - LongRuntimeValues addO = c.testScript("times_ten"); - assertThat(c.collect(addO.rangeQuery("foo", 1, 2), addO), equalTo(List.of())); + LongRuntimeValues timesTen = c.testScript("times_ten"); + assertThat(c.collect(timesTen.rangeQuery("foo", 1, 2), timesTen), equalTo(List.of())); visited.clear(); - assertThat(c.collect(addO.rangeQuery("foo", 9, 11), addO), equalTo(List.of(10L, 20L))); + assertThat(c.collect(timesTen.rangeQuery("foo", 9, 11), timesTen), equalTo(List.of(10L, 20L))); visited.clear(); - assertThat(c.collect(addO.rangeQuery("foo", 10, 11), addO), equalTo(List.of(10L, 20L))); + assertThat(c.collect(timesTen.rangeQuery("foo", 10, 11), timesTen), equalTo(List.of(10L, 20L))); visited.clear(); - assertThat(c.collect(addO.rangeQuery("foo", 19, 21), addO), equalTo(List.of(10L, 20L))); + assertThat(c.collect(timesTen.rangeQuery("foo", 19, 21), timesTen), equalTo(List.of(10L, 20L))); visited.clear(); - assertThat(c.collect(addO.rangeQuery("foo", 99, 101), addO), equalTo(List.of(100L, 200L))); + assertThat(c.collect(timesTen.rangeQuery("foo", 99, 101), timesTen), equalTo(List.of(100L, 200L))); } private TestCase randomLongs() throws IOException { diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java index a985b480a0632..5998160a794ba 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java @@ -123,7 +123,7 @@ protected V testScript(String name) throws IOException { return script(new Script(ScriptType.INLINE, "test", name, Map.of())); } - protected V script(Script script) throws IOException { + private V script(Script script) throws IOException { return newValues(scriptService.compile(script, scriptContext()), Map.of(), sourceLookup, fieldData); } @@ -131,6 +131,10 @@ protected List collect(String script) throws IOException { return collect(new MatchAllDocsQuery(), script(script)); } + protected List collect(V values) throws IOException { + return collect(new MatchAllDocsQuery(), values); + } + protected List collect(Query query, V values) throws IOException { // Now run the query and collect the results List result = new ArrayList<>(); From 216c02ce1d748c763d84af2f0677053004084820 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 30 Jun 2020 13:03:58 -0400 Subject: [PATCH 15/35] Range query --- .../runtimefields/StringRuntimeValues.java | 45 +++++++++++++++++++ .../StringScriptFieldScriptTests.java | 12 +++++ 2 files changed, 57 insertions(+) diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java index ce68d7bf51b48..09f2a7bb99b78 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java @@ -48,6 +48,10 @@ public Query prefixQuery(String fieldName, String value) { return unstarted().new PrefixQuery(fieldName, value); } + public Query rangeQuery(String fieldName, String lowerValue, String upperValue) { + return unstarted().new RangeQuery(fieldName, lowerValue, upperValue); + } + @Override protected SharedValues newSharedValues() { return new SharedValues(); @@ -204,5 +208,46 @@ public boolean equals(Object obj) { return prefix.equals(other.prefix); } } + + private class RangeQuery extends AbstractRuntimeQuery { + private final String lowerValue; + private final String upperValue; + + private RangeQuery(String fieldName, String lowerValue, String upperValue) { + super(fieldName); + this.lowerValue = lowerValue; + this.upperValue = upperValue; + assert lowerValue.compareTo(upperValue) <= 0; + } + + @Override + protected boolean matches() { + for (int i = 0; i < count; i++) { + if (lowerValue.compareTo(values[i]) <= 0 && upperValue.compareTo(values[i]) >= 0) { + return true; + } + } + return false; + } + + @Override + protected String bareToString() { + return "[" + lowerValue + "," + upperValue + "]"; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), lowerValue, upperValue); + } + + @Override + public boolean equals(Object obj) { + if (false == super.equals(obj)) { + return false; + } + RangeQuery other = (RangeQuery) obj; + return lowerValue.equals(other.lowerValue) && upperValue.equals(other.upperValue); + } + } } } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java index 1b8008f35980d..50675eeac2121 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java @@ -95,6 +95,18 @@ public void testPrefixQuery() throws IOException { assertThat(c.collect(addO.prefixQuery("foo", "d"), addO), equalTo(List.of("chickeno", "dogo"))); } + public void testRangeQuery() throws IOException { + TestCase c = multipleValuesInDocValues(); + StringRuntimeValues addO = c.testScript("add_o"); + assertThat(c.collect(addO.rangeQuery("foo", "catz", "cbat"), addO), equalTo(List.of())); + visited.clear(); + assertThat(c.collect(addO.rangeQuery("foo", "c", "cb"), addO), equalTo(List.of("cato", "pigo"))); + visited.clear(); + assertThat(c.collect(addO.rangeQuery("foo", "p", "q"), addO), equalTo(List.of("cato", "pigo"))); + visited.clear(); + assertThat(c.collect(addO.rangeQuery("foo", "doggie", "dogs"), addO), equalTo(List.of("chickeno", "dogo"))); + } + private TestCase randomStrings() throws IOException { return testCase(iw -> { iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef(randomAlphaOfLength(2))))); From 09e6febb6a0f9b165dea69fb3341e4af73aac8df Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 30 Jun 2020 13:24:35 -0400 Subject: [PATCH 16/35] Fuzzy --- .../runtimefields/StringRuntimeValues.java | 113 ++++++++++++++++++ .../StringScriptFieldScriptTests.java | 24 ++++ 2 files changed, 137 insertions(+) diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java index 09f2a7bb99b78..d15dfb54377ce 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java @@ -15,6 +15,7 @@ import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.RamUsageEstimator; import org.apache.lucene.util.automaton.ByteRunAutomaton; +import org.apache.lucene.util.automaton.CompiledAutomaton; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; @@ -40,10 +41,18 @@ public CheckedFunction do return unstarted().docValues(); } + public Query fuzzyQuery(String fieldName, String value, int maxEdits, int prefixLength, int maxExpansions, boolean transpositions) { + return unstarted().new FuzzyQuery(fieldName, value, maxEdits, prefixLength, maxExpansions, transpositions); + } + public Query termQuery(String fieldName, String value) { return unstarted().new TermQuery(fieldName, value); } + public Query termsQuery(String fieldName, String... values) { + return unstarted().new TermsQuery(fieldName, values); + } + public Query prefixQuery(String fieldName, String value) { return unstarted().new PrefixQuery(fieldName, value); } @@ -115,6 +124,64 @@ public BytesRef nextValue() throws IOException { } } + private class FuzzyQuery extends AbstractRuntimeQuery { + private final BytesRefBuilder scratch = new BytesRefBuilder(); + private final String term; + private final org.apache.lucene.search.FuzzyQuery delegate; + private final CompiledAutomaton automaton; + + private FuzzyQuery(String fieldName, String term, int maxEdits, int prefixLength, int maxExpansions, boolean transpositions) { + super(fieldName); + this.term = term; + delegate = new org.apache.lucene.search.FuzzyQuery( + new Term(fieldName, term), + maxEdits, + prefixLength, + maxExpansions, + transpositions + ); + automaton = delegate.getAutomata(); + if (automaton.type != CompiledAutomaton.AUTOMATON_TYPE.NORMAL) { + throw new IllegalArgumentException("Can't compile automaton for [" + delegate + "]"); + } + } + + @Override + protected boolean matches() { + for (int i = 0; i < count; i++) { + scratch.copyChars(values[i]); + if (automaton.runAutomaton.run(scratch.bytes(), 0, scratch.length())) { + return true; + } + } + return false; + } + + @Override + public void visit(QueryVisitor visitor) { + visitor.consumeTerms(this, new Term(fieldName, term)); + } + + @Override + protected String bareToString() { + return term + "~" + delegate.getMaxEdits(); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), term); + } + + @Override + public boolean equals(Object obj) { + if (false == super.equals(obj)) { + return false; + } + FuzzyQuery other = (FuzzyQuery) obj; + return delegate.equals(other.delegate); + } + } + private class TermQuery extends AbstractRuntimeQuery { private final String term; @@ -158,6 +225,52 @@ public boolean equals(Object obj) { } } + private class TermsQuery extends AbstractRuntimeQuery { + private final String[] terms; + + private TermsQuery(String fieldName, String[] terms) { + super(fieldName); + this.terms = terms.clone(); + Arrays.sort(terms); + } + + @Override + protected boolean matches() { + for (int i = 0; i < count; i++) { + if (Arrays.binarySearch(terms, values[i]) >= 0) { + return true; + } + } + return false; + } + + @Override + public void visit(QueryVisitor visitor) { + for (String term : terms) { + visitor.consumeTerms(this, new Term(fieldName, term)); + } + } + + @Override + protected String bareToString() { + return "{" + Arrays.toString(terms) + "}"; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), Arrays.hashCode(terms)); + } + + @Override + public boolean equals(Object obj) { + if (false == super.equals(obj)) { + return false; + } + TermsQuery other = (TermsQuery) obj; + return Arrays.equals(terms, other.terms); + } + } + private class PrefixQuery extends AbstractRuntimeQuery { private final String prefix; diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java index 50675eeac2121..a395eb390145b 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java @@ -69,6 +69,18 @@ public void testMultipleDocValuesValues() throws IOException { assertThat(multipleValuesInDocValues().collect(ADD_O), equalTo(List.of("cato", "pigo", "chickeno", "dogo"))); } + public void testFuzzyQuery() throws IOException { + TestCase c = multipleValuesInDocValues(); + StringRuntimeValues addO = c.testScript("add_o"); + assertThat(c.collect(addO.fuzzyQuery("foo", "caaaaat", 1, 1, 1, true), addO), equalTo(List.of())); + visited.clear(); + assertThat(c.collect(addO.fuzzyQuery("foo", "cat", 1, 1, 1, true), addO), equalTo(List.of("cato", "pigo"))); + visited.clear(); + assertThat(c.collect(addO.fuzzyQuery("foo", "pig", 1, 1, 1, true), addO), equalTo(List.of("cato", "pigo"))); + visited.clear(); + assertThat(c.collect(addO.fuzzyQuery("foo", "dog", 1, 1, 1, true), addO), equalTo(List.of("chickeno", "dogo"))); + } + public void testTermQuery() throws IOException { TestCase c = multipleValuesInDocValues(); StringRuntimeValues addO = c.testScript("add_o"); @@ -81,6 +93,18 @@ public void testTermQuery() throws IOException { assertThat(c.collect(addO.termQuery("foo", "dogo"), addO), equalTo(List.of("chickeno", "dogo"))); } + public void testTermsQuery() throws IOException { + TestCase c = multipleValuesInDocValues(); + StringRuntimeValues addO = c.testScript("add_o"); + assertThat(c.collect(addO.termsQuery("foo", "cat", "dog"), addO), equalTo(List.of())); + visited.clear(); + assertThat(c.collect(addO.termsQuery("foo", "cato", "piglet"), addO), equalTo(List.of("cato", "pigo"))); + visited.clear(); + assertThat(c.collect(addO.termsQuery("foo", "pigo", "catington"), addO), equalTo(List.of("cato", "pigo"))); + visited.clear(); + assertThat(c.collect(addO.termsQuery("foo", "dogo", "lightbulb"), addO), equalTo(List.of("chickeno", "dogo"))); + } + public void testPrefixQuery() throws IOException { TestCase c = multipleValuesInDocValues(); StringRuntimeValues addO = c.testScript("add_o"); From 4e66638182551ce467425822f0c953977e6b2520 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 30 Jun 2020 13:35:01 -0400 Subject: [PATCH 17/35] Wildcard --- .../runtimefields/StringRuntimeValues.java | 129 ++++++++++++------ .../StringScriptFieldScriptTests.java | 12 ++ 2 files changed, 103 insertions(+), 38 deletions(-) diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java index d15dfb54377ce..bf59ba5612e47 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java @@ -45,6 +45,10 @@ public Query fuzzyQuery(String fieldName, String value, int maxEdits, int prefix return unstarted().new FuzzyQuery(fieldName, value, maxEdits, prefixLength, maxExpansions, transpositions); } + public Query prefixQuery(String fieldName, String value) { + return unstarted().new PrefixQuery(fieldName, value); + } + public Query termQuery(String fieldName, String value) { return unstarted().new TermQuery(fieldName, value); } @@ -53,14 +57,14 @@ public Query termsQuery(String fieldName, String... values) { return unstarted().new TermsQuery(fieldName, values); } - public Query prefixQuery(String fieldName, String value) { - return unstarted().new PrefixQuery(fieldName, value); - } - public Query rangeQuery(String fieldName, String lowerValue, String upperValue) { return unstarted().new RangeQuery(fieldName, lowerValue, upperValue); } + public Query wildcardQuery(String fieldName, String pattern) { + return unstarted().new WildcardQuery(fieldName, pattern); + } + @Override protected SharedValues newSharedValues() { return new SharedValues(); @@ -182,6 +186,57 @@ public boolean equals(Object obj) { } } + private class PrefixQuery extends AbstractRuntimeQuery { + private final String prefix; + + private PrefixQuery(String fieldName, String prefix) { + super(fieldName); + this.prefix = prefix; + } + + @Override + protected boolean matches() { + for (int i = 0; i < count; i++) { + if (values[i].startsWith(prefix)) { + return true; + } + } + return false; + } + + @Override + protected String bareToString() { + return prefix + "*"; + } + + @Override + public void visit(QueryVisitor visitor) { + visitor.consumeTermsMatching( + this, + fieldName, + () -> new ByteRunAutomaton( + org.apache.lucene.search.PrefixQuery.toAutomaton(new BytesRef(prefix)), + true, + Integer.MAX_VALUE + ) + ); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), prefix); + } + + @Override + public boolean equals(Object obj) { + if (false == super.equals(obj)) { + return false; + } + PrefixQuery other = (PrefixQuery) obj; + return prefix.equals(other.prefix); + } + } + private class TermQuery extends AbstractRuntimeQuery { private final String term; @@ -271,18 +326,21 @@ public boolean equals(Object obj) { } } - private class PrefixQuery extends AbstractRuntimeQuery { - private final String prefix; + private class RangeQuery extends AbstractRuntimeQuery { + private final String lowerValue; + private final String upperValue; - private PrefixQuery(String fieldName, String prefix) { + private RangeQuery(String fieldName, String lowerValue, String upperValue) { super(fieldName); - this.prefix = prefix; + this.lowerValue = lowerValue; + this.upperValue = upperValue; + assert lowerValue.compareTo(upperValue) <= 0; } @Override protected boolean matches() { for (int i = 0; i < count; i++) { - if (values[i].startsWith(prefix)) { + if (lowerValue.compareTo(values[i]) <= 0 && upperValue.compareTo(values[i]) >= 0) { return true; } } @@ -291,25 +349,12 @@ protected boolean matches() { @Override protected String bareToString() { - return prefix + "*"; - } - - @Override - public void visit(QueryVisitor visitor) { - visitor.consumeTermsMatching( - this, - fieldName, - () -> new ByteRunAutomaton( - org.apache.lucene.search.PrefixQuery.toAutomaton(new BytesRef(prefix)), - true, - Integer.MAX_VALUE - ) - ); + return "[" + lowerValue + "," + upperValue + "]"; } @Override public int hashCode() { - return Objects.hash(super.hashCode(), prefix); + return Objects.hash(super.hashCode(), lowerValue, upperValue); } @Override @@ -317,26 +362,27 @@ public boolean equals(Object obj) { if (false == super.equals(obj)) { return false; } - PrefixQuery other = (PrefixQuery) obj; - return prefix.equals(other.prefix); + RangeQuery other = (RangeQuery) obj; + return lowerValue.equals(other.lowerValue) && upperValue.equals(other.upperValue); } } - private class RangeQuery extends AbstractRuntimeQuery { - private final String lowerValue; - private final String upperValue; + private class WildcardQuery extends AbstractRuntimeQuery { + private final BytesRefBuilder scratch = new BytesRefBuilder(); + private final String pattern; + private final ByteRunAutomaton automaton; - private RangeQuery(String fieldName, String lowerValue, String upperValue) { + private WildcardQuery(String fieldName, String pattern) { super(fieldName); - this.lowerValue = lowerValue; - this.upperValue = upperValue; - assert lowerValue.compareTo(upperValue) <= 0; + this.pattern = pattern; + automaton = new ByteRunAutomaton(org.apache.lucene.search.WildcardQuery.toAutomaton(new Term(fieldName, pattern))); } @Override protected boolean matches() { for (int i = 0; i < count; i++) { - if (lowerValue.compareTo(values[i]) <= 0 && upperValue.compareTo(values[i]) >= 0) { + scratch.copyChars(values[i]); + if (automaton.run(scratch.bytes(), 0, scratch.length())) { return true; } } @@ -345,12 +391,19 @@ protected boolean matches() { @Override protected String bareToString() { - return "[" + lowerValue + "," + upperValue + "]"; + return pattern; + } + + @Override + public void visit(QueryVisitor visitor) { + if (visitor.acceptField(fieldName)) { + visitor.consumeTermsMatching(this, fieldName, () -> automaton); + } } @Override public int hashCode() { - return Objects.hash(super.hashCode(), lowerValue, upperValue); + return Objects.hash(super.hashCode(), pattern); } @Override @@ -358,8 +411,8 @@ public boolean equals(Object obj) { if (false == super.equals(obj)) { return false; } - RangeQuery other = (RangeQuery) obj; - return lowerValue.equals(other.lowerValue) && upperValue.equals(other.upperValue); + WildcardQuery other = (WildcardQuery) obj; + return pattern.equals(other.pattern); } } } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java index a395eb390145b..90330a471288f 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java @@ -131,6 +131,18 @@ public void testRangeQuery() throws IOException { assertThat(c.collect(addO.rangeQuery("foo", "doggie", "dogs"), addO), equalTo(List.of("chickeno", "dogo"))); } + public void testWildcardQuery() throws IOException { + TestCase c = multipleValuesInDocValues(); + StringRuntimeValues addO = c.testScript("add_o"); + assertThat(c.collect(addO.wildcardQuery("foo", "cat"), addO), equalTo(List.of())); + visited.clear(); + assertThat(c.collect(addO.wildcardQuery("foo", "cat?"), addO), equalTo(List.of("cato", "pigo"))); + visited.clear(); + assertThat(c.collect(addO.wildcardQuery("foo", "p*"), addO), equalTo(List.of("cato", "pigo"))); + visited.clear(); + assertThat(c.collect(addO.wildcardQuery("foo", "do?o"), addO), equalTo(List.of("chickeno", "dogo"))); + } + private TestCase randomStrings() throws IOException { return testCase(iw -> { iw.addDocument(List.of(new SortedSetDocValuesField("foo", new BytesRef(randomAlphaOfLength(2))))); From 6837172e0aeec5f329caac0c0acb0188294e7789 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 30 Jun 2020 13:45:17 -0400 Subject: [PATCH 18/35] Regexp query --- .../runtimefields/StringRuntimeValues.java | 83 ++++++++++++++----- .../StringScriptFieldScriptTests.java | 15 +++- 2 files changed, 78 insertions(+), 20 deletions(-) diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java index bf59ba5612e47..c293a995b97c4 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java @@ -14,8 +14,10 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.BytesRefBuilder; import org.apache.lucene.util.RamUsageEstimator; +import org.apache.lucene.util.automaton.Automaton; import org.apache.lucene.util.automaton.ByteRunAutomaton; import org.apache.lucene.util.automaton.CompiledAutomaton; +import org.apache.lucene.util.automaton.RegExp; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; @@ -61,6 +63,10 @@ public Query rangeQuery(String fieldName, String lowerValue, String upperValue) return unstarted().new RangeQuery(fieldName, lowerValue, upperValue); } + public Query regexpQuery(String fieldName, String pattern, int flags, int maxDeterminizedStates) { + return unstarted().new RegexpQuery(fieldName, pattern, flags, maxDeterminizedStates); + } + public Query wildcardQuery(String fieldName, String pattern) { return unstarted().new WildcardQuery(fieldName, pattern); } @@ -146,6 +152,7 @@ private FuzzyQuery(String fieldName, String term, int maxEdits, int prefixLength ); automaton = delegate.getAutomata(); if (automaton.type != CompiledAutomaton.AUTOMATON_TYPE.NORMAL) { + // TODO I'll bet we have to actually implement all of these types throw new IllegalArgumentException("Can't compile automaton for [" + delegate + "]"); } } @@ -367,38 +374,48 @@ public boolean equals(Object obj) { } } - private class WildcardQuery extends AbstractRuntimeQuery { - private final BytesRefBuilder scratch = new BytesRefBuilder(); + private class RegexpQuery extends AbstractAutomatonQuery { private final String pattern; - private final ByteRunAutomaton automaton; + private final int flags; - private WildcardQuery(String fieldName, String pattern) { - super(fieldName); + private RegexpQuery(String fieldName, String pattern, int flags, int maxDeterminizedStates) { + super(fieldName, new RegExp(pattern, flags).toAutomaton(maxDeterminizedStates)); this.pattern = pattern; - automaton = new ByteRunAutomaton(org.apache.lucene.search.WildcardQuery.toAutomaton(new Term(fieldName, pattern))); + this.flags = flags; } @Override - protected boolean matches() { - for (int i = 0; i < count; i++) { - scratch.copyChars(values[i]); - if (automaton.run(scratch.bytes(), 0, scratch.length())) { - return true; - } - } - return false; + protected String bareToString() { + return "/" + pattern + "/"; } + @Override - protected String bareToString() { - return pattern; + public int hashCode() { + return Objects.hash(super.hashCode(), pattern, flags); } @Override - public void visit(QueryVisitor visitor) { - if (visitor.acceptField(fieldName)) { - visitor.consumeTermsMatching(this, fieldName, () -> automaton); + public boolean equals(Object obj) { + if (false == super.equals(obj)) { + return false; } + RegexpQuery other = (RegexpQuery) obj; + return pattern.equals(other.pattern) && flags == other.flags; + } + } + + private class WildcardQuery extends AbstractAutomatonQuery { + private final String pattern; + + private WildcardQuery(String fieldName, String pattern) { + super(fieldName, org.apache.lucene.search.WildcardQuery.toAutomaton(new Term(fieldName, pattern))); + this.pattern = pattern; + } + + @Override + protected String bareToString() { + return pattern; } @Override @@ -415,5 +432,33 @@ public boolean equals(Object obj) { return pattern.equals(other.pattern); } } + + private abstract class AbstractAutomatonQuery extends AbstractRuntimeQuery { + private final BytesRefBuilder scratch = new BytesRefBuilder(); + private final ByteRunAutomaton automaton; + + private AbstractAutomatonQuery(String fieldName, Automaton automaton) { + super(fieldName); + this.automaton = new ByteRunAutomaton(automaton); + } + + @Override + protected final boolean matches() { + for (int i = 0; i < count; i++) { + scratch.copyChars(values[i]); + if (automaton.run(scratch.bytes(), 0, scratch.length())) { + return true; + } + } + return false; + } + + @Override + public final void visit(QueryVisitor visitor) { + if (visitor.acceptField(fieldName)) { + visitor.consumeTermsMatching(this, fieldName, () -> automaton); + } + } + } } } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java index 90330a471288f..24215ad6e049b 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java @@ -11,10 +11,11 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.automaton.RegExp; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.plugins.ScriptPlugin; import org.elasticsearch.script.ScriptContext; @@ -131,6 +132,18 @@ public void testRangeQuery() throws IOException { assertThat(c.collect(addO.rangeQuery("foo", "doggie", "dogs"), addO), equalTo(List.of("chickeno", "dogo"))); } + public void testRegexpQuery() throws IOException { + TestCase c = multipleValuesInDocValues(); + StringRuntimeValues addO = c.testScript("add_o"); + assertThat(c.collect(addO.regexpQuery("foo", "cat", RegExp.ALL, 100000), addO), equalTo(List.of())); + visited.clear(); + assertThat(c.collect(addO.regexpQuery("foo", "cat[aeiou]", RegExp.ALL, 100000), addO), equalTo(List.of("cato", "pigo"))); + visited.clear(); + assertThat(c.collect(addO.regexpQuery("foo", "p.*", RegExp.ALL, 100000), addO), equalTo(List.of("cato", "pigo"))); + visited.clear(); + assertThat(c.collect(addO.regexpQuery("foo", "dog?o", RegExp.ALL, 100000), addO), equalTo(List.of("chickeno", "dogo"))); + } + public void testWildcardQuery() throws IOException { TestCase c = multipleValuesInDocValues(); StringRuntimeValues addO = c.testScript("add_o"); From f5106b2679751164c44c12571d6bc9c930140cb1 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 30 Jun 2020 14:00:13 -0400 Subject: [PATCH 19/35] Exists and some sorting --- .../runtimefields/DoubleRuntimeValues.java | 104 +++++++++++------- .../runtimefields/LongRuntimeValues.java | 104 +++++++++++------- .../runtimefields/StringRuntimeValues.java | 20 ++++ .../DoubleScriptFieldScriptTests.java | 32 ++++++ .../LongScriptFieldScriptTests.java | 32 ++++++ .../StringScriptFieldScriptTests.java | 31 ++++++ 6 files changed, 239 insertions(+), 84 deletions(-) diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleRuntimeValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleRuntimeValues.java index 9842fa8b69cc5..53b7038af8596 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleRuntimeValues.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleRuntimeValues.java @@ -36,6 +36,14 @@ public CheckedFunction 0; + } + + @Override + protected String bareToString() { + return "*"; + } + } + + private class RangeQuery extends AbstractRuntimeQuery { + private final double lowerValue; + private final double upperValue; + + private RangeQuery(String fieldName, double lowerValue, double upperValue) { super(fieldName); - this.term = term; + this.lowerValue = lowerValue; + this.upperValue = upperValue; } @Override protected boolean matches() { for (int i = 0; i < count; i++) { - if (term == values[i]) { + if (lowerValue <= values[i] && values[i] <= upperValue) { return true; } } return false; } - @Override - public void visit(QueryVisitor visitor) { - visitor.consumeTerms(this, new Term(fieldName, Double.toString(term))); - } - @Override protected String bareToString() { - return Double.toString(term); + return "[" + lowerValue + "," + upperValue + "]"; } @Override public int hashCode() { - return Objects.hash(super.hashCode(), term); + return Objects.hash(super.hashCode(), lowerValue, upperValue); } @Override @@ -144,24 +161,23 @@ public boolean equals(Object obj) { if (false == super.equals(obj)) { return false; } - TermQuery other = (TermQuery) obj; - return term == other.term; + RangeQuery other = (RangeQuery) obj; + return lowerValue == other.lowerValue && upperValue == other.upperValue; } } - private class TermsQuery extends AbstractRuntimeQuery { - private final double[] terms; + private class TermQuery extends AbstractRuntimeQuery { + private final double term; - private TermsQuery(String fieldName, double[] terms) { + private TermQuery(String fieldName, double term) { super(fieldName); - this.terms = terms.clone(); - Arrays.sort(terms); + this.term = term; } @Override protected boolean matches() { for (int i = 0; i < count; i++) { - if (Arrays.binarySearch(terms, values[i]) >= 0) { + if (term == values[i]) { return true; } } @@ -170,19 +186,17 @@ protected boolean matches() { @Override public void visit(QueryVisitor visitor) { - for (double term : terms) { - visitor.consumeTerms(this, new Term(fieldName, Double.toString(term))); - } + visitor.consumeTerms(this, new Term(fieldName, Double.toString(term))); } @Override protected String bareToString() { - return "{" + Arrays.toString(terms) + "}"; + return Double.toString(term); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), Arrays.hashCode(terms)); + return Objects.hash(super.hashCode(), term); } @Override @@ -190,39 +204,45 @@ public boolean equals(Object obj) { if (false == super.equals(obj)) { return false; } - TermsQuery other = (TermsQuery) obj; - return Arrays.equals(terms, other.terms); + TermQuery other = (TermQuery) obj; + return term == other.term; } } - private class RangeQuery extends AbstractRuntimeQuery { - private final double lowerValue; - private final double upperValue; + private class TermsQuery extends AbstractRuntimeQuery { + private final double[] terms; - private RangeQuery(String fieldName, double lowerValue, double upperValue) { + private TermsQuery(String fieldName, double[] terms) { super(fieldName); - this.lowerValue = lowerValue; - this.upperValue = upperValue; + this.terms = terms.clone(); + Arrays.sort(terms); } @Override protected boolean matches() { for (int i = 0; i < count; i++) { - if (lowerValue <= values[i] && values[i] <= upperValue) { + if (Arrays.binarySearch(terms, values[i]) >= 0) { return true; } } return false; } + @Override + public void visit(QueryVisitor visitor) { + for (double term : terms) { + visitor.consumeTerms(this, new Term(fieldName, Double.toString(term))); + } + } + @Override protected String bareToString() { - return "[" + lowerValue + "," + upperValue + "]"; + return "{" + Arrays.toString(terms) + "}"; } @Override public int hashCode() { - return Objects.hash(super.hashCode(), lowerValue, upperValue); + return Objects.hash(super.hashCode(), Arrays.hashCode(terms)); } @Override @@ -230,8 +250,8 @@ public boolean equals(Object obj) { if (false == super.equals(obj)) { return false; } - RangeQuery other = (RangeQuery) obj; - return lowerValue == other.lowerValue && upperValue == other.upperValue; + TermsQuery other = (TermsQuery) obj; + return Arrays.equals(terms, other.terms); } } } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongRuntimeValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongRuntimeValues.java index e1ae4ea3ee8a1..e0f63bd526e54 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongRuntimeValues.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongRuntimeValues.java @@ -36,6 +36,14 @@ public CheckedFunction d return unstarted().docValues(); } + public Query existsQuery(String fieldName) { + return unstarted().new ExistsQuery(fieldName); + } + + public Query rangeQuery(String fieldName, long lowerValue, long upperValue) { + return unstarted().new RangeQuery(fieldName, lowerValue, upperValue); + } + public Query termQuery(String fieldName, long value) { return unstarted().new TermQuery(fieldName, value); } @@ -44,10 +52,6 @@ public Query termsQuery(String fieldName, long... value) { return unstarted().new TermsQuery(fieldName, value); } - public Query rangeQuery(String fieldName, long lowerValue, long upperValue) { - return unstarted().new RangeQuery(fieldName, lowerValue, upperValue); - } - @Override protected SharedValues newSharedValues() { return new SharedValues(); @@ -134,37 +138,50 @@ public long cost() { } } - private class TermQuery extends AbstractRuntimeQuery { - private final long term; + private class ExistsQuery extends AbstractRuntimeQuery { + private ExistsQuery(String fieldName) { + super(fieldName); + } - private TermQuery(String fieldName, long term) { + @Override + protected boolean matches() { + return count > 0; + } + + @Override + protected String bareToString() { + return "*"; + } + } + + private class RangeQuery extends AbstractRuntimeQuery { + private final long lowerValue; + private final long upperValue; + + private RangeQuery(String fieldName, long lowerValue, long upperValue) { super(fieldName); - this.term = term; + this.lowerValue = lowerValue; + this.upperValue = upperValue; } @Override protected boolean matches() { for (int i = 0; i < count; i++) { - if (term == values[i]) { + if (lowerValue <= values[i] && values[i] <= upperValue) { return true; } } return false; } - @Override - public void visit(QueryVisitor visitor) { - visitor.consumeTerms(this, new Term(fieldName, Long.toString(term))); - } - @Override protected String bareToString() { - return Long.toString(term); + return "[" + lowerValue + "," + upperValue + "]"; } @Override public int hashCode() { - return Objects.hash(super.hashCode(), term); + return Objects.hash(super.hashCode(), lowerValue, upperValue); } @Override @@ -172,24 +189,23 @@ public boolean equals(Object obj) { if (false == super.equals(obj)) { return false; } - TermQuery other = (TermQuery) obj; - return term == other.term; + RangeQuery other = (RangeQuery) obj; + return lowerValue == other.lowerValue && upperValue == other.upperValue; } } - private class TermsQuery extends AbstractRuntimeQuery { - private final long[] terms; + private class TermQuery extends AbstractRuntimeQuery { + private final long term; - private TermsQuery(String fieldName, long[] terms) { + private TermQuery(String fieldName, long term) { super(fieldName); - this.terms = terms.clone(); - Arrays.sort(terms); + this.term = term; } @Override protected boolean matches() { for (int i = 0; i < count; i++) { - if (Arrays.binarySearch(terms, values[i]) >= 0) { + if (term == values[i]) { return true; } } @@ -198,19 +214,17 @@ protected boolean matches() { @Override public void visit(QueryVisitor visitor) { - for (long term : terms) { - visitor.consumeTerms(this, new Term(fieldName, Long.toString(term))); - } + visitor.consumeTerms(this, new Term(fieldName, Long.toString(term))); } @Override protected String bareToString() { - return "{" + Arrays.toString(terms) + "}"; + return Long.toString(term); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), Arrays.hashCode(terms)); + return Objects.hash(super.hashCode(), term); } @Override @@ -218,39 +232,45 @@ public boolean equals(Object obj) { if (false == super.equals(obj)) { return false; } - TermsQuery other = (TermsQuery) obj; - return Arrays.equals(terms, other.terms); + TermQuery other = (TermQuery) obj; + return term == other.term; } } - private class RangeQuery extends AbstractRuntimeQuery { - private final long lowerValue; - private final long upperValue; + private class TermsQuery extends AbstractRuntimeQuery { + private final long[] terms; - private RangeQuery(String fieldName, long lowerValue, long upperValue) { + private TermsQuery(String fieldName, long[] terms) { super(fieldName); - this.lowerValue = lowerValue; - this.upperValue = upperValue; + this.terms = terms.clone(); + Arrays.sort(terms); } @Override protected boolean matches() { for (int i = 0; i < count; i++) { - if (lowerValue <= values[i] && values[i] <= upperValue) { + if (Arrays.binarySearch(terms, values[i]) >= 0) { return true; } } return false; } + @Override + public void visit(QueryVisitor visitor) { + for (long term : terms) { + visitor.consumeTerms(this, new Term(fieldName, Long.toString(term))); + } + } + @Override protected String bareToString() { - return "[" + lowerValue + "," + upperValue + "]"; + return "{" + Arrays.toString(terms) + "}"; } @Override public int hashCode() { - return Objects.hash(super.hashCode(), lowerValue, upperValue); + return Objects.hash(super.hashCode(), Arrays.hashCode(terms)); } @Override @@ -258,8 +278,8 @@ public boolean equals(Object obj) { if (false == super.equals(obj)) { return false; } - RangeQuery other = (RangeQuery) obj; - return lowerValue == other.lowerValue && upperValue == other.upperValue; + TermsQuery other = (TermsQuery) obj; + return Arrays.equals(terms, other.terms); } } } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java index c293a995b97c4..18691c15dcd00 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java @@ -43,6 +43,10 @@ public CheckedFunction do return unstarted().docValues(); } + public Query existsQuery(String fieldName) { + return unstarted().new ExistsQuery(fieldName); + } + public Query fuzzyQuery(String fieldName, String value, int maxEdits, int prefixLength, int maxExpansions, boolean transpositions) { return unstarted().new FuzzyQuery(fieldName, value, maxEdits, prefixLength, maxExpansions, transpositions); } @@ -134,6 +138,22 @@ public BytesRef nextValue() throws IOException { } } + private class ExistsQuery extends AbstractRuntimeQuery { + private ExistsQuery(String fieldName) { + super(fieldName); + } + + @Override + protected boolean matches() { + return count > 0; + } + + @Override + protected String bareToString() { + return "*"; + } + } + private class FuzzyQuery extends AbstractRuntimeQuery { private final BytesRefBuilder scratch = new BytesRefBuilder(); private final String term; diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java index abd02fb90f242..6b518299df671 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java @@ -73,6 +73,12 @@ public void testMultipleDocValuesValues() throws IOException { ); } + public void testExistsQuery() throws IOException { + TestCase c = multipleValuesInDocValues(); + DoubleRuntimeValues isOnePointOne = c.testScript("is_one_point_one"); + assertThat(c.collect(isOnePointOne.existsQuery("foo"), isOnePointOne), equalTo(List.of(1.1))); + } + public void testTermQuery() throws IOException { TestCase c = multipleValuesInDocValues(); DoubleRuntimeValues timesTen = c.testScript("times_nine_point_nine"); @@ -237,6 +243,32 @@ public void execute() { return leafFactory; }; } + if (name.equals("is_one_point_one")) { + return (params, source, fieldData) -> { + DoubleScriptFieldScript.LeafFactory leafFactory = (ctx, sync) -> { + return new DoubleScriptFieldScript(params, source, fieldData, ctx, sync) { + @Override + protected void onSetDocument(int docId) { + int rebased = ctx.docBase + docId; + if (false == visited.add(rebased)) { + throw new AssertionError("Visited [" + rebased + "] twice. Order before was " + visited); + } + } + + @Override + public void execute() { + for (Object v : getDoc().get("foo")) { + double d = (double) v; + if (d == 1.1) { + sync.accept(1.1); + } + } + } + }; + }; + return leafFactory; + }; + } throw new IllegalArgumentException(); } }; diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java index 6c71fb0f4bd9a..95c50e4c671d0 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java @@ -66,6 +66,12 @@ public void testMultipleDocValuesValues() throws IOException { assertThat(multipleValuesInDocValues().collect(c.testScript("times_ten")), equalTo(List.of(10L, 20L, 100L, 200L))); } + public void testExistsQuery() throws IOException { + TestCase c = multipleValuesInDocValues(); + LongRuntimeValues isOne = c.testScript("is_one"); + assertThat(c.collect(isOne.existsQuery("foo"), isOne), equalTo(List.of(1L))); + } + public void testTermQuery() throws IOException { TestCase c = multipleValuesInDocValues(); LongRuntimeValues timesTen = c.testScript("times_ten"); @@ -224,6 +230,32 @@ public void execute() { return leafFactory; }; } + if (name.equals("is_one")) { + return (params, source, fieldData) -> { + LongScriptFieldScript.LeafFactory leafFactory = (ctx, sync) -> { + return new LongScriptFieldScript(params, source, fieldData, ctx, sync) { + @Override + protected void onSetDocument(int docId) { + int rebased = ctx.docBase + docId; + if (false == visited.add(rebased)) { + throw new AssertionError("Visited [" + rebased + "] twice. Order before was " + visited); + } + } + + @Override + public void execute() { + for (Object v : getDoc().get("foo")) { + long l = (long) v; + if (l == 1) { + sync.accept(1); + } + } + } + }; + }; + return leafFactory; + }; + } throw new IllegalArgumentException(); } }; diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java index 24215ad6e049b..0da26f7d6a677 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java @@ -70,6 +70,12 @@ public void testMultipleDocValuesValues() throws IOException { assertThat(multipleValuesInDocValues().collect(ADD_O), equalTo(List.of("cato", "pigo", "chickeno", "dogo"))); } + public void testExistsQuery() throws IOException { + TestCase c = multipleValuesInDocValues(); + StringRuntimeValues isCat = c.testScript("is_cat"); + assertThat(c.collect(isCat.existsQuery("foo"), isCat), equalTo(List.of("cat"))); + } + public void testFuzzyQuery() throws IOException { TestCase c = multipleValuesInDocValues(); StringRuntimeValues addO = c.testScript("add_o"); @@ -252,6 +258,31 @@ public void execute() { return leafFactory; }; } + if (name.equals("is_cat")) { + return (params, source, fieldData) -> { + StringScriptFieldScript.LeafFactory leafFactory = (ctx, sync) -> { + return new StringScriptFieldScript(params, source, fieldData, ctx, sync) { + @Override + protected void onSetDocument(int docId) { + int rebased = ctx.docBase + docId; + if (false == visited.add(rebased)) { + throw new AssertionError("Visited [" + rebased + "] twice. Order before was " + visited); + } + } + + @Override + public void execute() { + for (Object v : getDoc().get("foo")) { + if (v.equals("cat")) { + sync.accept("cat"); + } + } + } + }; + }; + return leafFactory; + }; + } throw new IllegalArgumentException(); } }; From 2e055218e94cb4ccf61b3f7118b83430a6271969 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 30 Jun 2020 14:03:50 -0400 Subject: [PATCH 20/35] Spotless --- .../elasticsearch/xpack/runtimefields/StringRuntimeValues.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java index 18691c15dcd00..efc7ac4752bae 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java @@ -409,7 +409,6 @@ protected String bareToString() { return "/" + pattern + "/"; } - @Override public int hashCode() { return Objects.hash(super.hashCode(), pattern, flags); From bfe15f835e3fe01928556e2a007b82fc4f33aac8 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 30 Jun 2020 15:43:00 -0400 Subject: [PATCH 21/35] WIP --- .../LongScriptFieldScriptTests.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java index 95c50e4c671d0..af36986410c5d 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java @@ -112,6 +112,24 @@ public void testRangeQuery() throws IOException { assertThat(c.collect(timesTen.rangeQuery("foo", 99, 101), timesTen), equalTo(List.of(100L, 200L))); } + /* + * TODO this should work but it doesn't because the BulkScorer scores a whole bunch at a time. + public void testInsideBoolTermQuery() throws IOException { + TestCase c = multipleValuesInDocValues(); + LongRuntimeValues timesTen = c.testScript("times_ten"); + assertThat( + c.collect( + new BooleanQuery.Builder().add(timesTen.termQuery("foo", 1), Occur.SHOULD) + .add(timesTen.termQuery("foo", 10), Occur.SHOULD) + .add(timesTen.termQuery("foo", 100), Occur.SHOULD) + .build(), + timesTen + ), + equalTo(List.of(10L, 20L, 100L, 200L)) + ); + } + */ + private TestCase randomLongs() throws IOException { return testCase(iw -> { iw.addDocument(List.of(new SortedNumericDocValuesField("foo", randomLong()))); From 7c18ddb51cbae8f55bb52513a2af900aebe0feff Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 1 Jul 2020 17:41:26 -0400 Subject: [PATCH 22/35] Fixup bool --- .../runtimefields/AbstractRuntimeValues.java | 2 +- .../ForceNoBulkScoringQuery.java | 113 ++++++++++++++++++ .../LongScriptFieldScriptTests.java | 23 ++-- 3 files changed, 130 insertions(+), 8 deletions(-) create mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/ForceNoBulkScoringQuery.java diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractRuntimeValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractRuntimeValues.java index 0b0ea2d67bf00..8832f1ecafa04 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractRuntimeValues.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractRuntimeValues.java @@ -107,7 +107,7 @@ protected final int maxDoc() { protected abstract void sort(); - public abstract class AbstractRuntimeQuery extends Query { + protected abstract class AbstractRuntimeQuery extends Query { protected final String fieldName; protected AbstractRuntimeQuery(String fieldName) { diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/ForceNoBulkScoringQuery.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/ForceNoBulkScoringQuery.java new file mode 100644 index 0000000000000..518b4fb9a06df --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/ForceNoBulkScoringQuery.java @@ -0,0 +1,113 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BulkScorer; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Matches; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.Weight; + +import java.io.IOException; +import java.util.Objects; +import java.util.Set; + +/** + * Forces wrapped queries to score documents one at a time which is important + * for runtime queries so they can more in lock step with one another. + *

+ * Inspired by the ForceNoBulkScoringQuery in Lucene's monitor project. + */ +class ForceNoBulkScoringQuery extends Query { + private final Query delegate; + + ForceNoBulkScoringQuery(Query inner) { + this.delegate = inner; + } + + @Override + public Query rewrite(IndexReader reader) throws IOException { + Query rewritten = delegate.rewrite(reader); + if (rewritten != delegate) { + return new ForceNoBulkScoringQuery(rewritten); + } + return this; + } + + @Override + public void visit(QueryVisitor visitor) { + delegate.visit(visitor); + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) { + return false; + } + ForceNoBulkScoringQuery that = (ForceNoBulkScoringQuery) o; + return Objects.equals(delegate, that.delegate); + } + + @Override + public int hashCode() { + return Objects.hash(delegate); + } + + @Override + public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { + Weight innerWeight = delegate.createWeight(searcher, scoreMode, boost); + return new Weight(this) { + @Override + public BulkScorer bulkScorer(LeafReaderContext context) throws IOException { + /* + * Intentionally do not delegate this to the innerWeight so + * we don't use its BulkScorer. This is the magic that causes + * us to skip bulk scoring. + */ + return super.bulkScorer(context); + } + + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return innerWeight.isCacheable(ctx); + } + + @Override + public Explanation explain(LeafReaderContext leafReaderContext, int i) throws IOException { + return innerWeight.explain(leafReaderContext, i); + } + + @Override + public Scorer scorer(LeafReaderContext leafReaderContext) throws IOException { + return innerWeight.scorer(leafReaderContext); + } + + @Override + public Matches matches(LeafReaderContext context, int doc) throws IOException { + return innerWeight.matches(context, doc); + } + + @Override + @Deprecated + public void extractTerms(Set terms) { + innerWeight.extractTerms(terms); + } + }; + } + + @Override + public String toString(String s) { + return "NoBulkScorer(" + delegate.toString(s) + ")"; + } +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java index af36986410c5d..9925e54521194 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java @@ -10,6 +10,8 @@ import org.apache.lucene.document.StoredField; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.search.BooleanClause.Occur; +import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.settings.Settings; @@ -112,23 +114,30 @@ public void testRangeQuery() throws IOException { assertThat(c.collect(timesTen.rangeQuery("foo", 99, 101), timesTen), equalTo(List.of(100L, 200L))); } - /* - * TODO this should work but it doesn't because the BulkScorer scores a whole bunch at a time. public void testInsideBoolTermQuery() throws IOException { + /* + * Its required that bool queries that contain more our runtime + * fields queries be wrapped in ForceNoBulkScoringQuery. Exactly what + * queries in the tree need to be wrapped and when isn't super clear + * but it is safest to wrap the whole query tree when there are *any* + * of these queries in it. We might be able to skip some of them + * eventually, when we're more comfortable with this. + */ TestCase c = multipleValuesInDocValues(); LongRuntimeValues timesTen = c.testScript("times_ten"); assertThat( c.collect( - new BooleanQuery.Builder().add(timesTen.termQuery("foo", 1), Occur.SHOULD) - .add(timesTen.termQuery("foo", 10), Occur.SHOULD) - .add(timesTen.termQuery("foo", 100), Occur.SHOULD) - .build(), + new ForceNoBulkScoringQuery( + new BooleanQuery.Builder().add(timesTen.termQuery("foo", 1), Occur.SHOULD) + .add(timesTen.termQuery("foo", 10), Occur.SHOULD) + .add(timesTen.termQuery("foo", 100), Occur.SHOULD) + .build() + ), timesTen ), equalTo(List.of(10L, 20L, 100L, 200L)) ); } - */ private TestCase randomLongs() throws IOException { return testCase(iw -> { From 79d41c86ee755f40f3fb344a279e4cffe4cfbe9a Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 2 Jul 2020 12:07:25 -0400 Subject: [PATCH 23/35] Remove some duplication --- .../AbstractScriptFieldScript.java | 4 +- .../DoubleScriptFieldScriptTests.java | 113 +++++++--------- .../LongScriptFieldScriptTests.java | 123 +++++++----------- .../ScriptFieldScriptTestCase.java | 17 ++- .../StringScriptFieldScriptTests.java | 122 ++++++----------- 5 files changed, 154 insertions(+), 225 deletions(-) diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldScript.java index 2ac6790424889..af832fd5f0b16 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldScript.java @@ -38,10 +38,10 @@ public AbstractScriptFieldScript(Map params, SourceLookup source public final void setDocId(int docId) { source.setSegmentAndDocument(ctx, docId); fieldData.setDocument(docId); - onSetDocument(docId); + onSetDocId(docId); } - protected void onSetDocument(int docId) {} + protected void onSetDocId(int docId) {} /** * Expose the {@code params} of the script to the script itself. diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java index 6b518299df671..75a2e86084a70 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java @@ -6,6 +6,17 @@ package org.elasticsearch.xpack.runtimefields; +import static org.hamcrest.Matchers.equalTo; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import java.util.function.DoubleConsumer; + import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.IndexableField; @@ -14,6 +25,7 @@ import org.apache.lucene.util.NumericUtils; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.SortedNumericDoubleValues; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType; @@ -25,24 +37,12 @@ import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript.Factory; -import java.io.IOException; -import java.util.Collection; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Consumer; - -import static org.hamcrest.Matchers.equalTo; - public class DoubleScriptFieldScriptTests extends ScriptFieldScriptTestCase< DoubleScriptFieldScript.Factory, DoubleRuntimeValues, SortedNumericDoubleValues, Double> { - private final Set visited = new LinkedHashSet<>(); - public void testConstant() throws IOException { assertThat(randomDoubles().collect("value(3.14)"), equalTo(List.of(3.14, 3.14))); } @@ -83,11 +83,8 @@ public void testTermQuery() throws IOException { TestCase c = multipleValuesInDocValues(); DoubleRuntimeValues timesTen = c.testScript("times_nine_point_nine"); assertThat(c.collect(timesTen.termQuery("foo", 1), timesTen), equalTo(List.of())); - visited.clear(); assertThat(c.collect(timesTen.termQuery("foo", 10.89), timesTen), equalTo(List.of(10.89, 21.78))); - visited.clear(); assertThat(c.collect(timesTen.termQuery("foo", 21.78), timesTen), equalTo(List.of(10.89, 21.78))); - visited.clear(); assertThat(c.collect(timesTen.termQuery("foo", 99.99), timesTen), equalTo(List.of(99.99, 198.99))); } @@ -95,13 +92,9 @@ public void testTermsQuery() throws IOException { TestCase c = multipleValuesInDocValues(); DoubleRuntimeValues timesTen = c.testScript("times_nine_point_nine"); assertThat(c.collect(timesTen.termsQuery("foo", 1, 2), timesTen), equalTo(List.of())); - visited.clear(); assertThat(c.collect(timesTen.termsQuery("foo", 10.89, 11), timesTen), equalTo(List.of(10.89, 21.78))); - visited.clear(); assertThat(c.collect(timesTen.termsQuery("foo", 21.78, 22), timesTen), equalTo(List.of(10.89, 21.78))); - visited.clear(); assertThat(c.collect(timesTen.termsQuery("foo", 20, 21.78), timesTen), equalTo(List.of(10.89, 21.78))); - visited.clear(); assertThat(c.collect(timesTen.termsQuery("foo", 99.99, 100), timesTen), equalTo(List.of(99.99, 198.99))); } @@ -109,15 +102,10 @@ public void testRangeQuery() throws IOException { TestCase c = multipleValuesInDocValues(); DoubleRuntimeValues timesTen = c.testScript("times_nine_point_nine"); assertThat(c.collect(timesTen.rangeQuery("foo", 1, 2), timesTen), equalTo(List.of())); - visited.clear(); assertThat(c.collect(timesTen.rangeQuery("foo", 9, 11), timesTen), equalTo(List.of(10.89, 21.78))); - visited.clear(); assertThat(c.collect(timesTen.rangeQuery("foo", 10.89, 11), timesTen), equalTo(List.of(10.89, 21.78))); - visited.clear(); assertThat(c.collect(timesTen.rangeQuery("foo", 21.78, 22), timesTen), equalTo(List.of(10.89, 21.78))); - visited.clear(); assertThat(c.collect(timesTen.rangeQuery("foo", 21, 21.78), timesTen), equalTo(List.of(10.89, 21.78))); - visited.clear(); assertThat(c.collect(timesTen.rangeQuery("foo", 99, 100), timesTen), equalTo(List.of(99.99, 198.99))); } @@ -221,53 +209,21 @@ public FactoryType compile( private DoubleScriptFieldScript.Factory compile(String name) { if (name.equals("times_nine_point_nine")) { - return (params, source, fieldData) -> { - DoubleScriptFieldScript.LeafFactory leafFactory = (ctx, sync) -> { - return new DoubleScriptFieldScript(params, source, fieldData, ctx, sync) { - @Override - protected void onSetDocument(int docId) { - int rebased = ctx.docBase + docId; - if (false == visited.add(rebased)) { - throw new AssertionError("Visited [" + rebased + "] twice. Order before was " + visited); - } - } - - @Override - public void execute() { - for (Object v : getDoc().get("foo")) { - sync.accept(((double) v) * 9.9); - } - } - }; - }; - return leafFactory; - }; + return assertingScript((fieldData, sync) -> { + for (Object v : fieldData.get("foo")) { + sync.accept(((double) v) * 9.9); + } + }); } if (name.equals("is_one_point_one")) { - return (params, source, fieldData) -> { - DoubleScriptFieldScript.LeafFactory leafFactory = (ctx, sync) -> { - return new DoubleScriptFieldScript(params, source, fieldData, ctx, sync) { - @Override - protected void onSetDocument(int docId) { - int rebased = ctx.docBase + docId; - if (false == visited.add(rebased)) { - throw new AssertionError("Visited [" + rebased + "] twice. Order before was " + visited); - } - } - - @Override - public void execute() { - for (Object v : getDoc().get("foo")) { - double d = (double) v; - if (d == 1.1) { - sync.accept(1.1); - } - } - } - }; - }; - return leafFactory; - }; + return assertingScript((fieldData, sync) -> { + for (Object v : fieldData.get("foo")) { + double d = (double) v; + if (d == 1.1) { + sync.accept(1.1); + } + } + }); } throw new IllegalArgumentException(); } @@ -275,4 +231,23 @@ public void execute() { } }); } + + private DoubleScriptFieldScript.Factory assertingScript(BiConsumer>, DoubleConsumer> impl) { + return (params, source, fieldData) -> { + DoubleScriptFieldScript.LeafFactory leafFactory = (ctx, sync) -> { + return new DoubleScriptFieldScript(params, source, fieldData, ctx, sync) { + @Override + public void execute() { + impl.accept(getDoc(), sync); + } + + @Override + protected void onSetDocId(int docId) { + onVisitDocId(ctx, docId); + } + }; + }; + return leafFactory; + }; + } } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java index 9925e54521194..93b9947b6048f 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java @@ -6,6 +6,17 @@ package org.elasticsearch.xpack.runtimefields; +import static org.hamcrest.Matchers.equalTo; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import java.util.function.LongConsumer; + import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.LeafReaderContext; @@ -15,6 +26,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberFieldType; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; @@ -25,24 +37,12 @@ import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.xpack.runtimefields.LongScriptFieldScript.Factory; -import java.io.IOException; -import java.util.Collection; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Consumer; - -import static org.hamcrest.Matchers.equalTo; - public class LongScriptFieldScriptTests extends ScriptFieldScriptTestCase< LongScriptFieldScript.Factory, LongRuntimeValues, SortedNumericDocValues, Long> { - private final Set visited = new LinkedHashSet<>(); - public void testConstant() throws IOException { assertThat(randomLongs().collect("value(10)"), equalTo(List.of(10L, 10L))); } @@ -78,11 +78,8 @@ public void testTermQuery() throws IOException { TestCase c = multipleValuesInDocValues(); LongRuntimeValues timesTen = c.testScript("times_ten"); assertThat(c.collect(timesTen.termQuery("foo", 1), timesTen), equalTo(List.of())); - visited.clear(); assertThat(c.collect(timesTen.termQuery("foo", 10), timesTen), equalTo(List.of(10L, 20L))); - visited.clear(); assertThat(c.collect(timesTen.termQuery("foo", 20), timesTen), equalTo(List.of(10L, 20L))); - visited.clear(); assertThat(c.collect(timesTen.termQuery("foo", 100), timesTen), equalTo(List.of(100L, 200L))); } @@ -90,13 +87,9 @@ public void testTermsQuery() throws IOException { TestCase c = multipleValuesInDocValues(); LongRuntimeValues timesTen = c.testScript("times_ten"); assertThat(c.collect(timesTen.termsQuery("foo", 1, 2), timesTen), equalTo(List.of())); - visited.clear(); assertThat(c.collect(timesTen.termsQuery("foo", 10, 11), timesTen), equalTo(List.of(10L, 20L))); - visited.clear(); assertThat(c.collect(timesTen.termsQuery("foo", 20, 21), timesTen), equalTo(List.of(10L, 20L))); - visited.clear(); assertThat(c.collect(timesTen.termsQuery("foo", 19, 20), timesTen), equalTo(List.of(10L, 20L))); - visited.clear(); assertThat(c.collect(timesTen.termsQuery("foo", 100, 11), timesTen), equalTo(List.of(100L, 200L))); } @@ -104,24 +97,19 @@ public void testRangeQuery() throws IOException { TestCase c = multipleValuesInDocValues(); LongRuntimeValues timesTen = c.testScript("times_ten"); assertThat(c.collect(timesTen.rangeQuery("foo", 1, 2), timesTen), equalTo(List.of())); - visited.clear(); assertThat(c.collect(timesTen.rangeQuery("foo", 9, 11), timesTen), equalTo(List.of(10L, 20L))); - visited.clear(); assertThat(c.collect(timesTen.rangeQuery("foo", 10, 11), timesTen), equalTo(List.of(10L, 20L))); - visited.clear(); assertThat(c.collect(timesTen.rangeQuery("foo", 19, 21), timesTen), equalTo(List.of(10L, 20L))); - visited.clear(); assertThat(c.collect(timesTen.rangeQuery("foo", 99, 101), timesTen), equalTo(List.of(100L, 200L))); } public void testInsideBoolTermQuery() throws IOException { /* - * Its required that bool queries that contain more our runtime - * fields queries be wrapped in ForceNoBulkScoringQuery. Exactly what - * queries in the tree need to be wrapped and when isn't super clear - * but it is safest to wrap the whole query tree when there are *any* - * of these queries in it. We might be able to skip some of them - * eventually, when we're more comfortable with this. + * Its required that bool queries that contain more our runtime fields queries + * be wrapped in ForceNoBulkScoringQuery. Exactly what queries in the tree need + * to be wrapped and when isn't super clear but it is safest to wrap the whole + * query tree when there are *any* of these queries in it. We might be able to + * skip some of them eventually, when we're more comfortable with this. */ TestCase c = multipleValuesInDocValues(); LongRuntimeValues timesTen = c.testScript("times_ten"); @@ -235,53 +223,21 @@ public FactoryType compile( private LongScriptFieldScript.Factory compile(String name) { if (name.equals("times_ten")) { - return (params, source, fieldData) -> { - LongScriptFieldScript.LeafFactory leafFactory = (ctx, sync) -> { - return new LongScriptFieldScript(params, source, fieldData, ctx, sync) { - @Override - protected void onSetDocument(int docId) { - int rebased = ctx.docBase + docId; - if (false == visited.add(rebased)) { - throw new AssertionError("Visited [" + rebased + "] twice. Order before was " + visited); - } - } - - @Override - public void execute() { - for (Object v : getDoc().get("foo")) { - sync.accept(((long) v) * 10); - } - } - }; - }; - return leafFactory; - }; + return assertingScript((fieldData, sync) -> { + for (Object v : fieldData.get("foo")) { + sync.accept(((long) v) * 10); + } + }); } if (name.equals("is_one")) { - return (params, source, fieldData) -> { - LongScriptFieldScript.LeafFactory leafFactory = (ctx, sync) -> { - return new LongScriptFieldScript(params, source, fieldData, ctx, sync) { - @Override - protected void onSetDocument(int docId) { - int rebased = ctx.docBase + docId; - if (false == visited.add(rebased)) { - throw new AssertionError("Visited [" + rebased + "] twice. Order before was " + visited); - } - } - - @Override - public void execute() { - for (Object v : getDoc().get("foo")) { - long l = (long) v; - if (l == 1) { - sync.accept(1); - } - } - } - }; - }; - return leafFactory; - }; + return assertingScript((fieldData, sync) -> { + for (Object v : fieldData.get("foo")) { + long l = (long) v; + if (l == 1) { + sync.accept(1); + } + } + }); } throw new IllegalArgumentException(); } @@ -289,4 +245,23 @@ public void execute() { } }); } + + private LongScriptFieldScript.Factory assertingScript(BiConsumer>, LongConsumer> impl) { + return (params, source, fieldData) -> { + LongScriptFieldScript.LeafFactory leafFactory = (ctx, sync) -> { + return new LongScriptFieldScript(params, source, fieldData, ctx, sync) { + @Override + public void execute() { + impl.accept(getDoc(), sync); + } + + @Override + protected void onSetDocId(int docId) { + onVisitDocId(ctx, docId); + } + }; + }; + return leafFactory; + }; + } } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java index 5998160a794ba..bd8fdf3b6aa71 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java @@ -45,8 +45,10 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Consumer; import java.util.function.Function; @@ -54,6 +56,7 @@ import static org.mockito.Mockito.when; public abstract class ScriptFieldScriptTestCase extends ESTestCase { + private final Set visited = new LinkedHashSet<>(); private final List lazyClose = new ArrayList<>(); private final ScriptService scriptService; @@ -159,6 +162,7 @@ public void collect(int docId) throws IOException { }; } }); + resetVisitedDocIds(); return result; } } @@ -177,7 +181,18 @@ private IndexSettings indexSettings() { @After public void closeAll() throws IOException { - Collections.reverse(lazyClose); // Close in the oppposite order added so readers close before directory + Collections.reverse(lazyClose); // Close in the reverse order added so readers close before directory IOUtils.close(lazyClose); } + + protected final void onVisitDocId(LeafReaderContext ctx, int docId) { + int rebased = ctx.docBase + docId; + if (false == visited.add(rebased)) { + throw new AssertionError("Visited [" + rebased + "] twice. Order before was " + visited); + } + } + + private void resetVisitedDocIds() { + visited.clear(); + } } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java index 0da26f7d6a677..a73d245f0c6d4 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java @@ -6,6 +6,17 @@ package org.elasticsearch.xpack.runtimefields; +import static org.hamcrest.Matchers.equalTo; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.BiConsumer; +import java.util.function.Consumer; + import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.IndexableField; @@ -14,6 +25,7 @@ import org.apache.lucene.util.automaton.RegExp; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.mapper.KeywordFieldMapper.KeywordFieldType; import org.elasticsearch.index.mapper.MappedFieldType; @@ -24,25 +36,12 @@ import org.elasticsearch.search.lookup.SourceLookup; import org.elasticsearch.xpack.runtimefields.StringScriptFieldScript.Factory; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Consumer; - -import static org.hamcrest.Matchers.equalTo; - public class StringScriptFieldScriptTests extends ScriptFieldScriptTestCase< StringScriptFieldScript.Factory, StringRuntimeValues, SortedBinaryDocValues, String> { - private final Set visited = new LinkedHashSet<>(); - public void testConstant() throws IOException { assertThat(randomStrings().collect("value('cat')"), equalTo(List.of("cat", "cat"))); } @@ -80,11 +79,8 @@ public void testFuzzyQuery() throws IOException { TestCase c = multipleValuesInDocValues(); StringRuntimeValues addO = c.testScript("add_o"); assertThat(c.collect(addO.fuzzyQuery("foo", "caaaaat", 1, 1, 1, true), addO), equalTo(List.of())); - visited.clear(); assertThat(c.collect(addO.fuzzyQuery("foo", "cat", 1, 1, 1, true), addO), equalTo(List.of("cato", "pigo"))); - visited.clear(); assertThat(c.collect(addO.fuzzyQuery("foo", "pig", 1, 1, 1, true), addO), equalTo(List.of("cato", "pigo"))); - visited.clear(); assertThat(c.collect(addO.fuzzyQuery("foo", "dog", 1, 1, 1, true), addO), equalTo(List.of("chickeno", "dogo"))); } @@ -92,11 +88,8 @@ public void testTermQuery() throws IOException { TestCase c = multipleValuesInDocValues(); StringRuntimeValues addO = c.testScript("add_o"); assertThat(c.collect(addO.termQuery("foo", "cat"), addO), equalTo(List.of())); - visited.clear(); assertThat(c.collect(addO.termQuery("foo", "cato"), addO), equalTo(List.of("cato", "pigo"))); - visited.clear(); assertThat(c.collect(addO.termQuery("foo", "pigo"), addO), equalTo(List.of("cato", "pigo"))); - visited.clear(); assertThat(c.collect(addO.termQuery("foo", "dogo"), addO), equalTo(List.of("chickeno", "dogo"))); } @@ -104,11 +97,8 @@ public void testTermsQuery() throws IOException { TestCase c = multipleValuesInDocValues(); StringRuntimeValues addO = c.testScript("add_o"); assertThat(c.collect(addO.termsQuery("foo", "cat", "dog"), addO), equalTo(List.of())); - visited.clear(); assertThat(c.collect(addO.termsQuery("foo", "cato", "piglet"), addO), equalTo(List.of("cato", "pigo"))); - visited.clear(); assertThat(c.collect(addO.termsQuery("foo", "pigo", "catington"), addO), equalTo(List.of("cato", "pigo"))); - visited.clear(); assertThat(c.collect(addO.termsQuery("foo", "dogo", "lightbulb"), addO), equalTo(List.of("chickeno", "dogo"))); } @@ -116,13 +106,9 @@ public void testPrefixQuery() throws IOException { TestCase c = multipleValuesInDocValues(); StringRuntimeValues addO = c.testScript("add_o"); assertThat(c.collect(addO.prefixQuery("foo", "catdog"), addO), equalTo(List.of())); - visited.clear(); assertThat(c.collect(addO.prefixQuery("foo", "cat"), addO), equalTo(List.of("cato", "pigo"))); - visited.clear(); assertThat(c.collect(addO.prefixQuery("foo", "pig"), addO), equalTo(List.of("cato", "pigo"))); - visited.clear(); assertThat(c.collect(addO.prefixQuery("foo", "dogo"), addO), equalTo(List.of("chickeno", "dogo"))); - visited.clear(); assertThat(c.collect(addO.prefixQuery("foo", "d"), addO), equalTo(List.of("chickeno", "dogo"))); } @@ -130,11 +116,8 @@ public void testRangeQuery() throws IOException { TestCase c = multipleValuesInDocValues(); StringRuntimeValues addO = c.testScript("add_o"); assertThat(c.collect(addO.rangeQuery("foo", "catz", "cbat"), addO), equalTo(List.of())); - visited.clear(); assertThat(c.collect(addO.rangeQuery("foo", "c", "cb"), addO), equalTo(List.of("cato", "pigo"))); - visited.clear(); assertThat(c.collect(addO.rangeQuery("foo", "p", "q"), addO), equalTo(List.of("cato", "pigo"))); - visited.clear(); assertThat(c.collect(addO.rangeQuery("foo", "doggie", "dogs"), addO), equalTo(List.of("chickeno", "dogo"))); } @@ -142,11 +125,8 @@ public void testRegexpQuery() throws IOException { TestCase c = multipleValuesInDocValues(); StringRuntimeValues addO = c.testScript("add_o"); assertThat(c.collect(addO.regexpQuery("foo", "cat", RegExp.ALL, 100000), addO), equalTo(List.of())); - visited.clear(); assertThat(c.collect(addO.regexpQuery("foo", "cat[aeiou]", RegExp.ALL, 100000), addO), equalTo(List.of("cato", "pigo"))); - visited.clear(); assertThat(c.collect(addO.regexpQuery("foo", "p.*", RegExp.ALL, 100000), addO), equalTo(List.of("cato", "pigo"))); - visited.clear(); assertThat(c.collect(addO.regexpQuery("foo", "dog?o", RegExp.ALL, 100000), addO), equalTo(List.of("chickeno", "dogo"))); } @@ -154,11 +134,8 @@ public void testWildcardQuery() throws IOException { TestCase c = multipleValuesInDocValues(); StringRuntimeValues addO = c.testScript("add_o"); assertThat(c.collect(addO.wildcardQuery("foo", "cat"), addO), equalTo(List.of())); - visited.clear(); assertThat(c.collect(addO.wildcardQuery("foo", "cat?"), addO), equalTo(List.of("cato", "pigo"))); - visited.clear(); assertThat(c.collect(addO.wildcardQuery("foo", "p*"), addO), equalTo(List.of("cato", "pigo"))); - visited.clear(); assertThat(c.collect(addO.wildcardQuery("foo", "do?o"), addO), equalTo(List.of("chickeno", "dogo"))); } @@ -236,52 +213,20 @@ public FactoryType compile( private StringScriptFieldScript.Factory compile(String name) { if (name.equals("add_o")) { - return (params, source, fieldData) -> { - StringScriptFieldScript.LeafFactory leafFactory = (ctx, sync) -> { - return new StringScriptFieldScript(params, source, fieldData, ctx, sync) { - @Override - protected void onSetDocument(int docId) { - int rebased = ctx.docBase + docId; - if (false == visited.add(rebased)) { - throw new AssertionError("Visited [" + rebased + "] twice. Order before was " + visited); - } - } - - @Override - public void execute() { - for (Object v : getDoc().get("foo")) { - sync.accept(v + "o"); - } - } - }; - }; - return leafFactory; - }; + return assertingScript((fieldData, sync) -> { + for (Object v : fieldData.get("foo")) { + sync.accept(v + "o"); + } + }); } if (name.equals("is_cat")) { - return (params, source, fieldData) -> { - StringScriptFieldScript.LeafFactory leafFactory = (ctx, sync) -> { - return new StringScriptFieldScript(params, source, fieldData, ctx, sync) { - @Override - protected void onSetDocument(int docId) { - int rebased = ctx.docBase + docId; - if (false == visited.add(rebased)) { - throw new AssertionError("Visited [" + rebased + "] twice. Order before was " + visited); - } - } - - @Override - public void execute() { - for (Object v : getDoc().get("foo")) { - if (v.equals("cat")) { - sync.accept("cat"); - } - } - } - }; - }; - return leafFactory; - }; + return assertingScript((fieldData, sync) -> { + for (Object v : fieldData.get("foo")) { + if (v.equals("cat")) { + sync.accept("cat"); + } + } + }); } throw new IllegalArgumentException(); } @@ -290,6 +235,25 @@ public void execute() { }); } + private StringScriptFieldScript.Factory assertingScript(BiConsumer>, Consumer> impl) { + return (params, source, fieldData) -> { + StringScriptFieldScript.LeafFactory leafFactory = (ctx, sync) -> { + return new StringScriptFieldScript(params, source, fieldData, ctx, sync) { + @Override + public void execute() { + impl.accept(getDoc(), sync); + } + + @Override + protected void onSetDocId(int docId) { + onVisitDocId(ctx, docId); + } + }; + }; + return leafFactory; + }; + } + @Override protected MappedFieldType[] fieldTypes() { return new MappedFieldType[] { new KeywordFieldType("foo") }; From ab6834e95c986e6dd1e82d21e3d884c66e08d67f Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 2 Jul 2020 12:17:44 -0400 Subject: [PATCH 24/35] Javadoc --- .../xpack/runtimefields/AbstractScriptFieldScript.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldScript.java index af832fd5f0b16..4c0818f7ba43e 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldScript.java @@ -41,6 +41,9 @@ public final void setDocId(int docId) { onSetDocId(docId); } + /** + * Optional hook for the script to take extra actions when moving to a document. + */ protected void onSetDocId(int docId) {} /** From 4e83233118ed9570605ca203bf65d65ba2cef820 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 2 Jul 2020 12:26:45 -0400 Subject: [PATCH 25/35] Javadoc --- .../DoubleScriptFieldScript.java | 17 ++++++++++++++++ .../runtimefields/LongScriptFieldScript.java | 17 ++++++++++++++++ .../StringScriptFieldScript.java | 20 +++++++++++++++++++ 3 files changed, 54 insertions(+) diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java index 363ddec4a5248..d6e5358c22312 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java @@ -26,15 +26,32 @@ static List whitelist() { return List.of(WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "double_whitelist.txt")); } + /** + * Magic constant that painless needs to name the parameters. There aren't any so it is empty. + */ public static final String[] PARAMETERS = {}; + /** + * Factory for building instances of the script for a particular search context. + */ public interface Factory extends ScriptFactory { LeafFactory newFactory(Map params, SourceLookup source, DocLookup fieldData); } + /** + * Factory for building the script for a particular leaf or for building + * runtime values which manages the creation of doc values and queries. + */ public interface LeafFactory { + /** + * Build a new script. + */ DoubleScriptFieldScript newInstance(LeafReaderContext ctx, DoubleConsumer sync) throws IOException; + /** + * Build an {@link DoubleRuntimeValues} to manage creation of doc + * values and queries using the script. + */ default DoubleRuntimeValues runtimeValues() throws IOException { return new DoubleRuntimeValues((ctx, sync) -> { DoubleScriptFieldScript script = newInstance(ctx, sync); diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java index 003d7ce78f15d..84f04c4b1c987 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java @@ -26,15 +26,32 @@ static List whitelist() { return List.of(WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "long_whitelist.txt")); } + /** + * Magic constant that painless needs to name the parameters. There aren't any so it is empty. + */ public static final String[] PARAMETERS = {}; + /** + * Factory for building instances of the script for a particular search context. + */ public interface Factory extends ScriptFactory { LeafFactory newFactory(Map params, SourceLookup source, DocLookup fieldData); } + /** + * Factory for building the script for a particular leaf or for building + * runtime values which manages the creation of doc values and queries. + */ public interface LeafFactory { + /** + * Build a new script. + */ LongScriptFieldScript newInstance(LeafReaderContext ctx, LongConsumer sync) throws IOException; + /** + * Build an {@link LongRuntimeValues} to manage creation of doc + * values and queries using the script. + */ default LongRuntimeValues runtimeValues() throws IOException { return new LongRuntimeValues((ctx, sync) -> { LongScriptFieldScript script = newInstance(ctx, sync); diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java index 801864cb3eef2..87e6cbcb56c46 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java @@ -19,6 +19,9 @@ import java.util.Map; import java.util.function.Consumer; +/** + * Script for building {@link String} values at runtime. + */ public abstract class StringScriptFieldScript extends AbstractScriptFieldScript { static final ScriptContext CONTEXT = new ScriptContext<>("string_script_field", Factory.class); @@ -26,15 +29,32 @@ static List whitelist() { return List.of(WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "string_whitelist.txt")); } + /** + * Magic constant that painless needs to name the parameters. There aren't any so it is empty. + */ public static final String[] PARAMETERS = {}; + /** + * Factory for building instances of the script for a particular search context. + */ public interface Factory extends ScriptFactory { LeafFactory newFactory(Map params, SourceLookup source, DocLookup fieldData); } + /** + * Factory for building the script for a particular leaf or for building + * runtime values which manages the creation of doc values and queries. + */ public interface LeafFactory { + /** + * Build a new script. + */ StringScriptFieldScript newInstance(LeafReaderContext ctx, Consumer sync) throws IOException; + /** + * Build an {@link StringRuntimeValues} to manage creation of doc + * values and queries using the script. + */ default StringRuntimeValues runtimeValues() throws IOException { return new StringRuntimeValues((ctx, sync) -> { StringScriptFieldScript script = newInstance(ctx, sync); From 810568ba8af9732e432a841e04d91bf64f617ade Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 2 Jul 2020 14:46:36 -0400 Subject: [PATCH 26/35] Javadoc --- .../elasticsearch/xpack/runtimefields/DoubleRuntimeValues.java | 3 +++ .../elasticsearch/xpack/runtimefields/LongRuntimeValues.java | 3 +++ .../elasticsearch/xpack/runtimefields/StringRuntimeValues.java | 3 +++ 3 files changed, 9 insertions(+) diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleRuntimeValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleRuntimeValues.java index 53b7038af8596..b8be31179a51e 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleRuntimeValues.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleRuntimeValues.java @@ -20,6 +20,9 @@ import java.util.function.DoubleConsumer; import java.util.function.IntConsumer; +/** + * Manages the creation of doc values and queries for {@code double} fields. + */ public final class DoubleRuntimeValues extends AbstractRuntimeValues { @FunctionalInterface public interface NewLeafLoader { diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongRuntimeValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongRuntimeValues.java index e0f63bd526e54..9a18e3fdd9faf 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongRuntimeValues.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongRuntimeValues.java @@ -20,6 +20,9 @@ import java.util.function.IntConsumer; import java.util.function.LongConsumer; +/** + * Manages the creation of doc values and queries for {@code long} fields. + */ public final class LongRuntimeValues extends AbstractRuntimeValues { @FunctionalInterface public interface NewLeafLoader { diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java index efc7ac4752bae..4f21956071a85 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java @@ -27,6 +27,9 @@ import java.util.function.Consumer; import java.util.function.IntConsumer; +/** + * Manages the creation of doc values and queries for {@link String} fields. + */ public final class StringRuntimeValues extends AbstractRuntimeValues { @FunctionalInterface public interface NewLeafLoader { From bc80a5e56a2ec0e0203c97b5b5e621318fd72d5d Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 2 Jul 2020 16:25:36 -0400 Subject: [PATCH 27/35] Wire in doc values and term query --- .../index/fielddata/SearchLookupAware.java | 2 + .../DoubleScriptFieldScript.java | 2 +- .../runtimefields/LongScriptFieldScript.java | 2 +- .../StringScriptFieldScript.java | 2 +- .../fielddata/ScriptBinaryDocValues.java | 41 --------- .../fielddata/ScriptBinaryFieldData.java | 34 +++----- .../mapper/RuntimeKeywordMappedFieldType.java | 20 ++++- .../test/runtime_fields/10_simple_script.yml | 84 +++++++++++++++++++ 8 files changed, 116 insertions(+), 71 deletions(-) delete mode 100644 x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryDocValues.java create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_simple_script.yml diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/SearchLookupAware.java b/server/src/main/java/org/elasticsearch/index/fielddata/SearchLookupAware.java index a7b33090ca6f9..4ed321c13ab70 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/SearchLookupAware.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/SearchLookupAware.java @@ -21,6 +21,8 @@ import org.elasticsearch.search.lookup.SearchLookup; +import java.io.IOException; + //TODO this is a temporary interface only to avoid changing signature of MappedFieldType#fielddataBuilder public interface SearchLookupAware { diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java index 47df82ea393e4..c5aadc82a4c3b 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java @@ -51,7 +51,7 @@ public interface LeafFactory { * Build an {@link DoubleRuntimeValues} to manage creation of doc * values and queries using the script. */ - default DoubleRuntimeValues runtimeValues() throws IOException { + default DoubleRuntimeValues runtimeValues() { return new DoubleRuntimeValues((ctx, sync) -> { DoubleScriptFieldScript script = newInstance(ctx, sync); return docId -> { diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java index fc48c5250fe47..0325d7b3cb744 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java @@ -51,7 +51,7 @@ public interface LeafFactory { * Build an {@link LongRuntimeValues} to manage creation of doc * values and queries using the script. */ - default LongRuntimeValues runtimeValues() throws IOException { + default LongRuntimeValues runtimeValues() { return new LongRuntimeValues((ctx, sync) -> { LongScriptFieldScript script = newInstance(ctx, sync); return docId -> { diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java index 98e0bc26b1e82..5d63073fc4cd2 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java @@ -54,7 +54,7 @@ public interface LeafFactory { * Build an {@link StringRuntimeValues} to manage creation of doc * values and queries using the script. */ - default StringRuntimeValues runtimeValues() throws IOException { + default StringRuntimeValues runtimeValues() { return new StringRuntimeValues((ctx, sync) -> { StringScriptFieldScript script = newInstance(ctx, sync); return docId -> { diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryDocValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryDocValues.java deleted file mode 100644 index d673ca1f3d881..0000000000000 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryDocValues.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -package org.elasticsearch.xpack.runtimefields.fielddata; - -import org.elasticsearch.index.fielddata.SortingBinaryDocValues; -import org.elasticsearch.xpack.runtimefields.StringScriptFieldScript; - -public final class ScriptBinaryDocValues extends SortingBinaryDocValues { - - private final StringScriptFieldScript script; - private final ScriptBinaryFieldData.ScriptBinaryResult scriptBinaryResult; - - ScriptBinaryDocValues(StringScriptFieldScript script, ScriptBinaryFieldData.ScriptBinaryResult scriptBinaryResult) { - this.script = script; - this.scriptBinaryResult = scriptBinaryResult; - } - - @Override - public boolean advanceExact(int doc) { - script.setDocId(doc); - script.execute(); - - count = scriptBinaryResult.getResult().size(); - if (count == 0) { - grow(); - return false; - } - - int i = 0; - for (String value : scriptBinaryResult.getResult()) { - grow(); - values[i++].copyChars(value); - } - sort(); - return true; - } -} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryFieldData.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryFieldData.java index 42ca35731274b..ec9108e32e7e9 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryFieldData.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryFieldData.java @@ -10,6 +10,7 @@ import org.apache.lucene.search.SortField; import org.apache.lucene.util.SetOnce; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.index.AbstractIndexComponent; import org.elasticsearch.index.IndexSettings; @@ -33,9 +34,7 @@ import org.elasticsearch.xpack.runtimefields.StringScriptFieldScript; import java.io.IOException; -import java.util.ArrayList; import java.util.Collections; -import java.util.List; public final class ScriptBinaryFieldData extends AbstractIndexComponent implements @@ -64,7 +63,7 @@ public IndexFieldData build( private final String fieldName; private final StringScriptFieldScript.Factory scriptFactory; - private final SetOnce leafFactory = new SetOnce<>(); + private final SetOnce> docValuesBuilder = new SetOnce<>(); private ScriptBinaryFieldData(IndexSettings indexSettings, String fieldName, StringScriptFieldScript.Factory scriptFactory) { super(indexSettings); @@ -72,9 +71,11 @@ private ScriptBinaryFieldData(IndexSettings indexSettings, String fieldName, Str this.scriptFactory = scriptFactory; } + @Override public void setSearchLookup(SearchLookup searchLookup) { // TODO wire the params from the mappings definition, we don't parse them yet - this.leafFactory.set(scriptFactory.newFactory(Collections.emptyMap(), searchLookup)); + // TODO it'd be nice if we could stuff `runtimeValues` some place into the search context so we could reuse it + this.docValuesBuilder.set(scriptFactory.newFactory(Collections.emptyMap(), searchLookup).runtimeValues().docValues()); } @Override @@ -102,10 +103,7 @@ public ScriptBinaryLeafFieldData load(LeafReaderContext context) { @Override public ScriptBinaryLeafFieldData loadDirect(LeafReaderContext context) throws IOException { - ScriptBinaryResult scriptBinaryResult = new ScriptBinaryResult(); - return new ScriptBinaryLeafFieldData( - new ScriptBinaryDocValues(leafFactory.get().newInstance(context, scriptBinaryResult::accept), scriptBinaryResult) - ); + return new ScriptBinaryLeafFieldData(docValuesBuilder.get().apply(context)); } @Override @@ -134,10 +132,10 @@ public void clear() { } static class ScriptBinaryLeafFieldData implements LeafFieldData { - private final ScriptBinaryDocValues scriptBinaryDocValues; + private final SortedBinaryDocValues docValues; - ScriptBinaryLeafFieldData(ScriptBinaryDocValues scriptBinaryDocValues) { - this.scriptBinaryDocValues = scriptBinaryDocValues; + ScriptBinaryLeafFieldData(SortedBinaryDocValues docValues) { + this.docValues = docValues; } @Override @@ -147,7 +145,7 @@ public ScriptDocValues getScriptValues() { @Override public SortedBinaryDocValues getBytesValues() { - return scriptBinaryDocValues; + return docValues; } @Override @@ -160,16 +158,4 @@ public void close() { } } - - static class ScriptBinaryResult { - private final List result = new ArrayList<>(); - - void accept(String value) { - this.result.add(value); - } - - List getResult() { - return result; - } - } } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldType.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldType.java index 5aef8e857408a..33337c1b6cb02 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldType.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldType.java @@ -6,10 +6,11 @@ package org.elasticsearch.xpack.runtimefields.mapper; +import org.apache.logging.log4j.LogManager; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.ToXContent.Params; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.TextSearchInfo; @@ -27,7 +28,7 @@ public final class RuntimeKeywordMappedFieldType extends MappedFieldType { private final StringScriptFieldScript.Factory scriptFactory; RuntimeKeywordMappedFieldType(String name, Script script, StringScriptFieldScript.Factory scriptFactory, Map meta) { - super(name, false, false, TextSearchInfo.NONE, meta); + super(name, false, false, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); this.script = script; this.scriptFactory = scriptFactory; } @@ -60,15 +61,28 @@ public String typeName() { return ScriptFieldMapper.CONTENT_TYPE; } + @Override public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { // TODO once we get SearchLookup as an argument, we can already call scriptFactory.newFactory here and pass through the result return new ScriptBinaryFieldData.Builder(scriptFactory); } + private String toValue(Object value) { + if (value == null) { + return null; + } + if (value instanceof BytesRef) { + return ((BytesRef) value).utf8ToString(); + } + return value.toString(); + } + @Override public Query termQuery(Object value, QueryShardContext context) { - return null; + // TODO cache the runtimeValues in the context somehow + LogManager.getLogger().error("ASDFDAF [{}]", toValue(value)); + return scriptFactory.newFactory(script.getParams(), context.lookup()).runtimeValues().termQuery(name(), toValue(value)); } @Override diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_simple_script.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_simple_script.yml new file mode 100644 index 0000000000000..f795166998db3 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_simple_script.yml @@ -0,0 +1,84 @@ +--- +setup: + - do: + indices.create: + index: sensor + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + timestamp: + type: date + temperature: + type: long + voltage: + type: float + node: + type: keyword + + - do: + bulk: + index: sensor + refresh: true + body: | + {"index":{}} + {"timestamp": 1516729294000, "temperature": 200, "voltage": 5.2, "node": "a"} + {"index":{}} + {"timestamp": 1516642894000, "temperature": 201, "voltage": 5.8, "node": "b"} + {"index":{}} + {"timestamp": 1516556494000, "temperature": 202, "voltage": 5.1, "node": "a"} + {"index":{}} + {"timestamp": 1516470094000, "temperature": 198, "voltage": 5.6, "node": "b"} + {"index":{}} + {"timestamp": 1516383694000, "temperature": 200, "voltage": 4.2, "node": "c"} + {"index":{}} + {"timestamp": 1516297294000, "temperature": 202, "voltage": 4.0, "node": "c"} + +--- +keyword: + - do: + indices.put_mapping: + index: sensor + body: + properties: + day_of_week: + type: script + runtime_type: keyword + script: value(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT)) + + - do: + search: + index: sensor + body: + docvalue_fields: + - day_of_week + sort: timestamp + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.day_of_week: [Thursday]} + - match: {hits.hits.1.fields.day_of_week: [Friday]} + - match: {hits.hits.2.fields.day_of_week: [Saturday]} + - match: {hits.hits.3.fields.day_of_week: [Sunday]} + - match: {hits.hits.4.fields.day_of_week: [Monday]} + - match: {hits.hits.5.fields.day_of_week: [Tuesday]} + + - do: + search: + index: sensor + body: + query: + term: + day_of_week: Monday + - match: {hits.total.value: 1} + - match: {hits.hits.0._source.voltage: 5.8} + + - do: + search: + index: sensor + body: + query: + match: + day_of_week: Monday + - match: {hits.total.value: 1} + - match: {hits.hits.0._source.voltage: 5.8} From 93a031f3294bb9f2481b76dcc16a71eb46548cfd Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 2 Jul 2020 17:51:14 -0400 Subject: [PATCH 28/35] Integrate! --- .../search/lookup/DocLookup.java | 6 +- .../search/lookup/LeafDocLookup.java | 19 +- .../search/lookup/SearchLookup.java | 2 +- .../search/lookup/LeafDocLookupTests.java | 2 +- .../runtimefields/StringRuntimeValues.java | 32 ++-- .../fielddata/ScriptBinaryFieldData.java | 2 +- .../mapper/RuntimeKeywordMappedFieldType.java | 82 +++++++-- .../StringScriptFieldScriptTests.java | 12 +- .../test/runtime_fields/10_keyword_script.yml | 172 ++++++++++++++++++ .../test/runtime_fields/10_simple_script.yml | 84 --------- 10 files changed, 291 insertions(+), 122 deletions(-) create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword_script.yml delete mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_simple_script.yml diff --git a/server/src/main/java/org/elasticsearch/search/lookup/DocLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/DocLookup.java index 1f0e53fca14c9..7fd6a21adc8ec 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/DocLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/DocLookup.java @@ -29,10 +29,12 @@ public class DocLookup { private final MapperService mapperService; private final Function> fieldDataLookup; + private final SearchLookup searchLookup; - public DocLookup(MapperService mapperService, Function> fieldDataLookup) { + public DocLookup(MapperService mapperService, Function> fieldDataLookup, SearchLookup searchLookup) { this.mapperService = mapperService; this.fieldDataLookup = fieldDataLookup; + this.searchLookup = searchLookup; } public MapperService mapperService() { @@ -44,6 +46,6 @@ public IndexFieldData getForField(MappedFieldType fieldType) { } public LeafDocLookup getLeafDocLookup(LeafReaderContext context) { - return new LeafDocLookup(mapperService, fieldDataLookup, context); + return new LeafDocLookup(mapperService, fieldDataLookup, searchLookup, context); } } diff --git a/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java index e06f1981f099d..791ae06e1a3c2 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java @@ -21,7 +21,9 @@ import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.LeafFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.index.fielddata.SearchLookupAware; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; @@ -41,14 +43,20 @@ public class LeafDocLookup implements Map> { private final MapperService mapperService; private final Function> fieldDataLookup; + private final SearchLookup searchLookup; private final LeafReaderContext reader; private int docId = -1; - LeafDocLookup(MapperService mapperService, Function> fieldDataLookup, - LeafReaderContext reader) { + LeafDocLookup( + MapperService mapperService, + Function> fieldDataLookup, + SearchLookup searchLookup, + LeafReaderContext reader + ) { this.mapperService = mapperService; this.fieldDataLookup = fieldDataLookup; + this.searchLookup = searchLookup; this.reader = reader; } @@ -75,7 +83,12 @@ public ScriptDocValues get(Object key) { scriptValues = AccessController.doPrivileged(new PrivilegedAction>() { @Override public ScriptDocValues run() { - return fieldDataLookup.apply(fieldType).load(reader).getScriptValues(); + // TODO should this go through QueryShardContext? + IndexFieldData ifd = fieldDataLookup.apply(fieldType); + if (ifd instanceof SearchLookupAware) { + ((SearchLookupAware) ifd).setSearchLookup(searchLookup); + } + return ifd.load(reader).getScriptValues(); } }); localCacheFieldData.put(fieldName, scriptValues); diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java index 04aef7d2e8f63..b723406c31ae7 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java @@ -35,7 +35,7 @@ public class SearchLookup { final FieldsLookup fieldsLookup; public SearchLookup(MapperService mapperService, Function> fieldDataLookup) { - docMap = new DocLookup(mapperService, fieldDataLookup); + docMap = new DocLookup(mapperService, fieldDataLookup, this); sourceLookup = new SourceLookup(); fieldsLookup = new FieldsLookup(mapperService); } diff --git a/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java b/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java index 6aaa4a1ed6a58..8dbc73767fbee 100644 --- a/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java +++ b/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java @@ -53,7 +53,7 @@ public void setUp() throws Exception { docLookup = new LeafDocLookup(mapperService, ignored -> fieldData, - null); + null, null); } public void testBasicLookup() { diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java index 4f21956071a85..b4e490a63256d 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java @@ -58,20 +58,20 @@ public Query prefixQuery(String fieldName, String value) { return unstarted().new PrefixQuery(fieldName, value); } - public Query termQuery(String fieldName, String value) { - return unstarted().new TermQuery(fieldName, value); + public Query rangeQuery(String fieldName, String lowerValue, String upperValue, boolean includeLower, boolean includeUpper) { + return unstarted().new RangeQuery(fieldName, lowerValue, upperValue, includeLower, includeUpper); } - public Query termsQuery(String fieldName, String... values) { - return unstarted().new TermsQuery(fieldName, values); + public Query regexpQuery(String fieldName, String pattern, int flags, int maxDeterminizedStates) { + return unstarted().new RegexpQuery(fieldName, pattern, flags, maxDeterminizedStates); } - public Query rangeQuery(String fieldName, String lowerValue, String upperValue) { - return unstarted().new RangeQuery(fieldName, lowerValue, upperValue); + public Query termQuery(String fieldName, String value) { + return unstarted().new TermQuery(fieldName, value); } - public Query regexpQuery(String fieldName, String pattern, int flags, int maxDeterminizedStates) { - return unstarted().new RegexpQuery(fieldName, pattern, flags, maxDeterminizedStates); + public Query termsQuery(String fieldName, String... values) { + return unstarted().new TermsQuery(fieldName, values); } public Query wildcardQuery(String fieldName, String pattern) { @@ -359,19 +359,29 @@ public boolean equals(Object obj) { private class RangeQuery extends AbstractRuntimeQuery { private final String lowerValue; private final String upperValue; + private final boolean includeLower; + private final boolean includeUpper; - private RangeQuery(String fieldName, String lowerValue, String upperValue) { + private RangeQuery(String fieldName, String lowerValue, String upperValue, boolean includeLower, boolean includeUpper) { super(fieldName); this.lowerValue = lowerValue; this.upperValue = upperValue; + this.includeLower = includeLower; + this.includeUpper = includeUpper; assert lowerValue.compareTo(upperValue) <= 0; } @Override protected boolean matches() { for (int i = 0; i < count; i++) { - if (lowerValue.compareTo(values[i]) <= 0 && upperValue.compareTo(values[i]) >= 0) { - return true; + int lct = lowerValue.compareTo(values[i]); + boolean lowerOk = includeLower ? lct <= 0 : lct < 0; + if (lowerOk) { + int uct = upperValue.compareTo(values[i]); + boolean upperOk = includeUpper ? uct >= 0 : uct > 0; + if (upperOk) { + return true; + } } } return false; diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryFieldData.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryFieldData.java index ec9108e32e7e9..bb76003595bcc 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryFieldData.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryFieldData.java @@ -36,7 +36,7 @@ import java.io.IOException; import java.util.Collections; -public final class ScriptBinaryFieldData extends AbstractIndexComponent +public final class ScriptBinaryFieldData extends AbstractIndexComponent // TODO do we need to extends AbstractIndexComponent? implements IndexFieldData, SearchLookupAware { diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldType.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldType.java index 33337c1b6cb02..4d618fc46fa45 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldType.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldType.java @@ -6,9 +6,13 @@ package org.elasticsearch.xpack.runtimefields.mapper; -import org.apache.logging.log4j.LogManager; import org.apache.lucene.search.Query; +import org.apache.lucene.search.MultiTermQuery.RewriteMethod; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.geo.ShapeRelation; +import org.elasticsearch.common.lucene.BytesRefs; +import org.elasticsearch.common.time.DateMathParser; +import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.xcontent.ToXContent.Params; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.fielddata.IndexFieldData; @@ -16,10 +20,13 @@ import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.script.Script; +import org.elasticsearch.xpack.runtimefields.StringRuntimeValues; import org.elasticsearch.xpack.runtimefields.StringScriptFieldScript; import org.elasticsearch.xpack.runtimefields.fielddata.ScriptBinaryFieldData; import java.io.IOException; +import java.time.ZoneId; +import java.util.List; import java.util.Map; public final class RuntimeKeywordMappedFieldType extends MappedFieldType { @@ -61,33 +68,78 @@ public String typeName() { return ScriptFieldMapper.CONTENT_TYPE; } + private StringRuntimeValues runtimeValues(QueryShardContext ctx) { + // TODO cache the runtimeValues in the context somehow + return scriptFactory.newFactory(script.getParams(), ctx.lookup()).runtimeValues(); + } @Override public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { - // TODO once we get SearchLookup as an argument, we can already call scriptFactory.newFactory here and pass through the result + // TODO figure out how to use runtimeValues here return new ScriptBinaryFieldData.Builder(scriptFactory); } - private String toValue(Object value) { - if (value == null) { - return null; - } - if (value instanceof BytesRef) { - return ((BytesRef) value).utf8ToString(); - } - return value.toString(); + @Override + public Query existsQuery(QueryShardContext context) { + return runtimeValues(context).existsQuery(name()); + } + + @Override + public Query fuzzyQuery( + Object value, + Fuzziness fuzziness, + int prefixLength, + int maxExpansions, + boolean transpositions, + QueryShardContext context + ) { + String term = BytesRefs.toString(value); + return runtimeValues(context).fuzzyQuery(name(), term, fuzziness.asDistance(term), prefixLength, maxExpansions, transpositions); + } + + @Override + public Query prefixQuery(String value, RewriteMethod method, QueryShardContext context) { + return runtimeValues(context).prefixQuery(name(), value); + } + + @Override + public Query rangeQuery( + Object lowerTerm, + Object upperTerm, + boolean includeLower, + boolean includeUpper, + ShapeRelation relation, + ZoneId timeZone, + DateMathParser parser, + QueryShardContext context + ) { + return runtimeValues(context).rangeQuery( + name(), + BytesRefs.toString(lowerTerm), + BytesRefs.toString(upperTerm), + includeLower, + includeUpper + ); + } + + @Override + public Query regexpQuery(String value, int flags, int maxDeterminizedStates, RewriteMethod method, QueryShardContext context) { + return runtimeValues(context).regexpQuery(name(), value, flags, maxDeterminizedStates); } @Override public Query termQuery(Object value, QueryShardContext context) { - // TODO cache the runtimeValues in the context somehow - LogManager.getLogger().error("ASDFDAF [{}]", toValue(value)); - return scriptFactory.newFactory(script.getParams(), context.lookup()).runtimeValues().termQuery(name(), toValue(value)); + return runtimeValues(context).termQuery(name(), BytesRefs.toString(value)); } @Override - public Query existsQuery(QueryShardContext context) { - return null; + public Query termsQuery(List values, QueryShardContext context) { + return runtimeValues(context).termsQuery(name(), values.stream().map(BytesRefs::toString).toArray(String[]::new)); + } + + @Override + public Query wildcardQuery(String value, RewriteMethod method, QueryShardContext context) { + return runtimeValues(context).wildcardQuery(name(), value); } void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java index 0b61c1cba1638..749733b93343b 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java @@ -114,10 +114,14 @@ public void testPrefixQuery() throws IOException { public void testRangeQuery() throws IOException { TestCase c = multipleValuesInDocValues(); StringRuntimeValues addO = c.testScript("add_o"); - assertThat(c.collect(addO.rangeQuery("foo", "catz", "cbat"), addO), equalTo(List.of())); - assertThat(c.collect(addO.rangeQuery("foo", "c", "cb"), addO), equalTo(List.of("cato", "pigo"))); - assertThat(c.collect(addO.rangeQuery("foo", "p", "q"), addO), equalTo(List.of("cato", "pigo"))); - assertThat(c.collect(addO.rangeQuery("foo", "doggie", "dogs"), addO), equalTo(List.of("chickeno", "dogo"))); + assertThat(c.collect(addO.rangeQuery("foo", "catz", "cbat", false, false), addO), equalTo(List.of())); + assertThat(c.collect(addO.rangeQuery("foo", "c", "cb", false, false), addO), equalTo(List.of("cato", "pigo"))); + assertThat(c.collect(addO.rangeQuery("foo", "p", "q", false, false), addO), equalTo(List.of("cato", "pigo"))); + assertThat(c.collect(addO.rangeQuery("foo", "doggie", "dogs", false, false), addO), equalTo(List.of("chickeno", "dogo"))); + assertThat(c.collect(addO.rangeQuery("foo", "dogo", "dogs", false, false), addO), equalTo(List.of())); + assertThat(c.collect(addO.rangeQuery("foo", "dogo", "dogs", true, false), addO), equalTo(List.of("chickeno", "dogo"))); + assertThat(c.collect(addO.rangeQuery("foo", "dog", "dogo", false, false), addO), equalTo(List.of())); + assertThat(c.collect(addO.rangeQuery("foo", "dog", "dogo", false, true), addO), equalTo(List.of("chickeno", "dogo"))); } public void testRegexpQuery() throws IOException { diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword_script.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword_script.yml new file mode 100644 index 0000000000000..db3ae6ee23c2b --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword_script.yml @@ -0,0 +1,172 @@ +--- +setup: + - do: + indices.create: + index: sensor + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + timestamp: + type: date + temperature: + type: long + voltage: + type: float + node: + type: keyword + day_of_week: + type: script + runtime_type: keyword + script: value(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT)) + days_starting_with_s: + type: script + runtime_type: keyword + script: | + String dayOfWeek = doc['day_of_week'].value; + if (dayOfWeek.startsWith('S')) { + value(dayOfWeek); + } + + - do: + bulk: + index: sensor + refresh: true + body: | + {"index":{}} + {"timestamp": 1516729294000, "temperature": 200, "voltage": 5.2, "node": "a"} + {"index":{}} + {"timestamp": 1516642894000, "temperature": 201, "voltage": 5.8, "node": "b"} + {"index":{}} + {"timestamp": 1516556494000, "temperature": 202, "voltage": 5.1, "node": "a"} + {"index":{}} + {"timestamp": 1516470094000, "temperature": 198, "voltage": 5.6, "node": "b"} + {"index":{}} + {"timestamp": 1516383694000, "temperature": 200, "voltage": 4.2, "node": "c"} + {"index":{}} + {"timestamp": 1516297294000, "temperature": 202, "voltage": 4.0, "node": "c"} + +--- +"fetch": + - do: + search: + index: sensor + body: + docvalue_fields: + - day_of_week + - days_starting_with_s + sort: timestamp + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.day_of_week: [Thursday]} + - match: {hits.hits.1.fields.day_of_week: [Friday]} + - match: {hits.hits.2.fields.day_of_week: [Saturday]} + - match: {hits.hits.3.fields.day_of_week: [Sunday]} + - match: {hits.hits.4.fields.day_of_week: [Monday]} + - match: {hits.hits.5.fields.day_of_week: [Tuesday]} + - is_false: hits.hits.0.fields.days_starting_with_s + - is_false: hits.hits.1.fields.days_starting_with_s + - match: {hits.hits.2.fields.days_starting_with_s: [Saturday]} + - match: {hits.hits.3.fields.days_starting_with_s: [Sunday]} + - is_false: hits.hits.4.fields.days_starting_with_s + - is_false: hits.hits.5.fields.days_starting_with_s + +--- +"exists query": + - do: + search: + index: sensor + body: + query: + exists: + field: days_starting_with_s + - match: {hits.total.value: 2} + +--- +"fuzzy query": + - do: + search: + index: sensor + body: + query: + fuzzy: + day_of_week: Mondai + - match: {hits.total.value: 1} + - match: {hits.hits.0._source.voltage: 5.8} + +--- +"match query": + - do: + search: + index: sensor + body: + query: + match: + day_of_week: Monday + - match: {hits.total.value: 1} + - match: {hits.hits.0._source.voltage: 5.8} + +--- +"prefix query": + - do: + search: + index: sensor + body: + query: + prefix: + day_of_week: Mon + - match: {hits.total.value: 1} + - match: {hits.hits.0._source.voltage: 5.8} + +--- +"regexp query": + - do: + search: + index: sensor + body: + query: + regexp: + day_of_week: + value: Mon.+ + sort: timestamp + - match: {hits.total.value: 1} + - match: {hits.hits.0._source.voltage: 5.8} + +--- +"term query": + - do: + search: + index: sensor + body: + query: + term: + day_of_week: Monday + - match: {hits.total.value: 1} + - match: {hits.hits.0._source.voltage: 5.8} + +--- +"terms query": + - do: + search: + index: sensor + body: + query: + terms: + day_of_week: [Monday, Tuesday] + sort: timestamp + - match: {hits.total.value: 2} + - match: {hits.hits.0._source.voltage: 5.8} + - match: {hits.hits.1._source.voltage: 5.2} + +--- +"wildcard query": + - do: + search: + index: sensor + body: + query: + wildcard: + day_of_week: Mond?y + - match: {hits.total.value: 1} + - match: {hits.hits.0._source.voltage: 5.8} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_simple_script.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_simple_script.yml deleted file mode 100644 index f795166998db3..0000000000000 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_simple_script.yml +++ /dev/null @@ -1,84 +0,0 @@ ---- -setup: - - do: - indices.create: - index: sensor - body: - settings: - number_of_shards: 1 - number_of_replicas: 0 - mappings: - properties: - timestamp: - type: date - temperature: - type: long - voltage: - type: float - node: - type: keyword - - - do: - bulk: - index: sensor - refresh: true - body: | - {"index":{}} - {"timestamp": 1516729294000, "temperature": 200, "voltage": 5.2, "node": "a"} - {"index":{}} - {"timestamp": 1516642894000, "temperature": 201, "voltage": 5.8, "node": "b"} - {"index":{}} - {"timestamp": 1516556494000, "temperature": 202, "voltage": 5.1, "node": "a"} - {"index":{}} - {"timestamp": 1516470094000, "temperature": 198, "voltage": 5.6, "node": "b"} - {"index":{}} - {"timestamp": 1516383694000, "temperature": 200, "voltage": 4.2, "node": "c"} - {"index":{}} - {"timestamp": 1516297294000, "temperature": 202, "voltage": 4.0, "node": "c"} - ---- -keyword: - - do: - indices.put_mapping: - index: sensor - body: - properties: - day_of_week: - type: script - runtime_type: keyword - script: value(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT)) - - - do: - search: - index: sensor - body: - docvalue_fields: - - day_of_week - sort: timestamp - - match: {hits.total.value: 6} - - match: {hits.hits.0.fields.day_of_week: [Thursday]} - - match: {hits.hits.1.fields.day_of_week: [Friday]} - - match: {hits.hits.2.fields.day_of_week: [Saturday]} - - match: {hits.hits.3.fields.day_of_week: [Sunday]} - - match: {hits.hits.4.fields.day_of_week: [Monday]} - - match: {hits.hits.5.fields.day_of_week: [Tuesday]} - - - do: - search: - index: sensor - body: - query: - term: - day_of_week: Monday - - match: {hits.total.value: 1} - - match: {hits.hits.0._source.voltage: 5.8} - - - do: - search: - index: sensor - body: - query: - match: - day_of_week: Monday - - match: {hits.total.value: 1} - - match: {hits.hits.0._source.voltage: 5.8} From 340962146efd86de80047e00711ee20a75124905 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 2 Jul 2020 18:01:34 -0400 Subject: [PATCH 29/35] Its over nine thousand! --- .../xpack/runtimefields/AbstractRuntimeValues.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractRuntimeValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractRuntimeValues.java index 8832f1ecafa04..52998204b5221 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractRuntimeValues.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractRuntimeValues.java @@ -135,8 +135,8 @@ public boolean matches() throws IOException { @Override public float matchCost() { - // TODO we have no idea what this should be and no real way to get one - return 1000f; + // TODO we don't have a good way of estimating the complexity of the script so we just go with 9000 + return approximation().cost() * 9000f; } }; return new ConstantScoreScorer(this, score(), scoreMode, twoPhase); From b5b02ed445609da253af69911eefa0b34edd2eac Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 2 Jul 2020 18:03:22 -0400 Subject: [PATCH 30/35] In a bool --- .../test/runtime_fields/10_keyword_script.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword_script.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword_script.yml index db3ae6ee23c2b..e0c5ef0db1640 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword_script.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/10_keyword_script.yml @@ -170,3 +170,21 @@ setup: day_of_week: Mond?y - match: {hits.total.value: 1} - match: {hits.hits.0._source.voltage: 5.8} + +--- +"in a bool query": + - do: + search: + index: sensor + body: + query: + bool: + should: + - term: + day_of_week: Monday + - term: + day_of_week: Tuesday + sort: timestamp + - match: {hits.total.value: 2} + - match: {hits.hits.0._source.voltage: 5.8} + - match: {hits.hits.1._source.voltage: 5.2} From e17fef8a7b416bb7369ad3e39122088b69232523 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 6 Jul 2020 09:41:14 -0400 Subject: [PATCH 31/35] itr --- .../org/elasticsearch/index/fielddata/SearchLookupAware.java | 2 -- .../runtimefields/mapper/RuntimeKeywordMappedFieldType.java | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/SearchLookupAware.java b/server/src/main/java/org/elasticsearch/index/fielddata/SearchLookupAware.java index 4ed321c13ab70..a7b33090ca6f9 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/SearchLookupAware.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/SearchLookupAware.java @@ -21,8 +21,6 @@ import org.elasticsearch.search.lookup.SearchLookup; -import java.io.IOException; - //TODO this is a temporary interface only to avoid changing signature of MappedFieldType#fielddataBuilder public interface SearchLookupAware { diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldType.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldType.java index 4d618fc46fa45..996acb71bd02e 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldType.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldType.java @@ -75,7 +75,7 @@ private StringRuntimeValues runtimeValues(QueryShardContext ctx) { @Override public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { - // TODO figure out how to use runtimeValues here + // TODO once we get SearchLookup as an argument, we can already call scriptFactory.newFactory here and pass through the result return new ScriptBinaryFieldData.Builder(scriptFactory); } From edc571fe0c69047ea7771d272ea1cf46aac341f1 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 8 Jul 2020 10:52:06 -0400 Subject: [PATCH 32/35] Drop shared iteration tests --- .../runtimefields/AbstractScriptFieldScript.java | 6 ------ .../DoubleScriptFieldScriptTests.java | 5 ----- .../runtimefields/LongScriptFieldScriptTests.java | 5 ----- .../runtimefields/ScriptFieldScriptTestCase.java | 15 --------------- .../StringScriptFieldScriptTests.java | 5 ----- 5 files changed, 36 deletions(-) diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldScript.java index cbedd2b94e1b0..fc4058f9c02d8 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractScriptFieldScript.java @@ -33,14 +33,8 @@ public AbstractScriptFieldScript(Map params, SearchLookup search */ public final void setDocId(int docId) { leafSearchLookup.setDocument(docId); - onSetDocId(docId); } - /** - * Optional hook for the script to take extra actions when moving to a document. - */ - protected void onSetDocId(int docId) {} - /** * Expose the {@code params} of the script to the script itself. */ diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java index 2cdddbf6da9a4..44dc32a793a34 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java @@ -238,11 +238,6 @@ private DoubleScriptFieldScript.Factory assertingScript(BiConsumer extends ESTestCase { - private final Set visited = new LinkedHashSet<>(); private final List lazyClose = new ArrayList<>(); private final ScriptService scriptService; @@ -160,7 +157,6 @@ public void collect(int docId) throws IOException { }; } }); - resetVisitedDocIds(); return result; } } @@ -182,15 +178,4 @@ public void closeAll() throws IOException { Collections.reverse(lazyClose); // Close in the reverse order added so readers close before directory IOUtils.close(lazyClose); } - - protected final void onVisitDocId(LeafReaderContext ctx, int docId) { - int rebased = ctx.docBase + docId; - if (false == visited.add(rebased)) { - throw new AssertionError("Visited [" + rebased + "] twice. Order before was " + visited); - } - } - - private void resetVisitedDocIds() { - visited.clear(); - } } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java index 749733b93343b..f8b9934334951 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java @@ -246,11 +246,6 @@ private StringScriptFieldScript.Factory assertingScript(BiConsumer Date: Wed, 8 Jul 2020 11:02:58 -0400 Subject: [PATCH 33/35] Remove caching --- .../runtimefields/AbstractRuntimeValues.java | 213 +++++++----------- ...ues.java => DoubleRuntimeFieldHelper.java} | 21 +- .../DoubleScriptFieldScript.java | 6 +- ...alues.java => LongRuntimeFieldHelper.java} | 21 +- .../runtimefields/LongScriptFieldScript.java | 6 +- ...ues.java => StringRuntimeFieldHelper.java} | 29 +-- .../StringScriptFieldScript.java | 6 +- .../mapper/RuntimeKeywordMappedFieldType.java | 4 +- .../DoubleScriptFieldScriptTests.java | 15 +- .../LongScriptFieldScriptTests.java | 16 +- .../ScriptFieldScriptTestCase.java | 18 +- .../StringScriptFieldScriptTests.java | 23 +- 12 files changed, 162 insertions(+), 216 deletions(-) rename x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/{DoubleRuntimeValues.java => DoubleRuntimeFieldHelper.java} (92%) rename x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/{LongRuntimeValues.java => LongRuntimeFieldHelper.java} (92%) rename x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/{StringRuntimeValues.java => StringRuntimeFieldHelper.java} (93%) diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractRuntimeValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractRuntimeValues.java index 52998204b5221..b9f81572e2177 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractRuntimeValues.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/AbstractRuntimeValues.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.runtimefields; import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.search.ConstantScoreScorer; import org.apache.lucene.search.ConstantScoreWeight; import org.apache.lucene.search.DocIdSetIterator; @@ -24,152 +23,112 @@ /** * Abstract base for implementing doc values and queries against values - * calculated at runtime. The tricky thing about this is that we'd like to - * calculate the values as few times as possible in case the calculation is - * expensive, but some of the APIs that we rely on to - * calculate the values like {@link SortedNumericDocValues#advanceExact(int)} - * are "forwards only". - *

- * We solve this in the same way that big cities handle public transportation: - * with a bus! In our case, the bus is subclasses of {@link SharedValues}. - * Queries and doc values are implemented calling {@link #unstarted()} to get - * the {@linkplain SharedValues} that has yet to start iterating. That way - * many queries can share the same underlying {@linkplain SharedValues} - * instance, only calculating the values for a document once. If other code - * needs to iterate the values after the first iteration has started then - * it'll get a new {@linkplain SharedValues} from {@linkplain #unstarted}, - * this "leaving on a different bus". - * - * @param the subtype of {@link SharedValues} needed by the subclass + * calculated at runtime. */ -public abstract class AbstractRuntimeValues.SharedValues> { - private SV unstarted; +public abstract class AbstractRuntimeValues { + protected int count; + private boolean sort; + + private int docId = -1; + private int maxDoc; + + protected final IntConsumer leafCursor(LeafReaderContext ctx) throws IOException { + IntConsumer leafLoader = newLeafLoader(ctx); + docId = -1; + maxDoc = ctx.reader().maxDoc(); + return new IntConsumer() { + @Override + public void accept(int targetDocId) { + if (docId == targetDocId) { + return; + } + docId = targetDocId; + count = 0; + leafLoader.accept(targetDocId); + if (sort) { + sort(); + } + } + }; + } - protected final SV unstarted() { - if (unstarted == null) { - unstarted = newSharedValues(); - } - return unstarted; + protected final void alwaysSortResults() { + sort = true; } - protected abstract SV newSharedValues(); + protected final int docId() { + return docId; + } - protected abstract class SharedValues { - protected int count; - private boolean sort; + protected final int maxDoc() { + return maxDoc; + } - private int lastDocBase = -1; - private IntConsumer lastLeafCursor; - private int docId = -1; - private int maxDoc; + protected abstract IntConsumer newLeafLoader(LeafReaderContext ctx) throws IOException; - protected final IntConsumer leafCursor(LeafReaderContext ctx) throws IOException { - if (lastDocBase != ctx.docBase) { - if (lastDocBase == -1) { - // Now that we're started future iterations can't share these values. - unstarted = null; - } - lastDocBase = ctx.docBase; - IntConsumer leafLoader = newLeafLoader(ctx); - docId = -1; - maxDoc = ctx.reader().maxDoc(); - lastLeafCursor = new IntConsumer() { - @Override - public void accept(int targetDocId) { - if (docId == targetDocId) { - return; - } - docId = targetDocId; - count = 0; - leafLoader.accept(targetDocId); - if (sort) { - sort(); - } - } - }; - } - return lastLeafCursor; - } + protected abstract void sort(); - protected final void alwaysSortResults() { - sort = true; - } - - protected final int docId() { - return docId; - } + protected abstract class AbstractRuntimeQuery extends Query { + protected final String fieldName; - protected final int maxDoc() { - return maxDoc; + protected AbstractRuntimeQuery(String fieldName) { + this.fieldName = fieldName; } - protected abstract IntConsumer newLeafLoader(LeafReaderContext ctx) throws IOException; - - protected abstract void sort(); - - protected abstract class AbstractRuntimeQuery extends Query { - protected final String fieldName; + @Override + public final Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { + return new ConstantScoreWeight(this, boost) { + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return false; // scripts aren't really cacheable at this point + } - protected AbstractRuntimeQuery(String fieldName) { - this.fieldName = fieldName; - } + @Override + public Scorer scorer(LeafReaderContext ctx) throws IOException { + IntConsumer leafCursor = leafCursor(ctx); + DocIdSetIterator approximation = DocIdSetIterator.all(ctx.reader().maxDoc()); + TwoPhaseIterator twoPhase = new TwoPhaseIterator(approximation) { + @Override + public boolean matches() throws IOException { + leafCursor.accept(approximation.docID()); + return AbstractRuntimeQuery.this.matches(); + } - @Override - public final Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { - return new ConstantScoreWeight(this, boost) { - @Override - public boolean isCacheable(LeafReaderContext ctx) { - return false; // scripts aren't really cacheable at this point - } - - @Override - public Scorer scorer(LeafReaderContext ctx) throws IOException { - IntConsumer leafCursor = leafCursor(ctx); - DocIdSetIterator approximation = DocIdSetIterator.all(ctx.reader().maxDoc()); - TwoPhaseIterator twoPhase = new TwoPhaseIterator(approximation) { - @Override - public boolean matches() throws IOException { - leafCursor.accept(approximation.docID()); - return AbstractRuntimeQuery.this.matches(); - } - - @Override - public float matchCost() { - // TODO we don't have a good way of estimating the complexity of the script so we just go with 9000 - return approximation().cost() * 9000f; - } - }; - return new ConstantScoreScorer(this, score(), scoreMode, twoPhase); - } - }; - } + @Override + public float matchCost() { + // TODO we don't have a good way of estimating the complexity of the script so we just go with 9000 + return approximation().cost() * 9000f; + } + }; + return new ConstantScoreScorer(this, score(), scoreMode, twoPhase); + } + }; + } - protected abstract boolean matches(); + protected abstract boolean matches(); - @Override - public final String toString(String field) { - if (fieldName.contentEquals(field)) { - return bareToString(); - } - return fieldName + ":" + bareToString(); + @Override + public final String toString(String field) { + if (fieldName.contentEquals(field)) { + return bareToString(); } + return fieldName + ":" + bareToString(); + } - protected abstract String bareToString(); + protected abstract String bareToString(); - @Override - public int hashCode() { - return Objects.hash(fieldName); - } + @Override + public int hashCode() { + return Objects.hash(fieldName); + } - @Override - public boolean equals(Object obj) { - if (obj == null || getClass() != obj.getClass()) { - return false; - } - @SuppressWarnings("unchecked") - AbstractRuntimeQuery other = (AbstractRuntimeQuery) obj; - return fieldName.equals(other.fieldName); + @Override + public boolean equals(Object obj) { + if (obj == null || getClass() != obj.getClass()) { + return false; } + AbstractRuntimeQuery other = (AbstractRuntimeQuery) obj; + return fieldName.equals(other.fieldName); } } } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleRuntimeValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleRuntimeFieldHelper.java similarity index 92% rename from x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleRuntimeValues.java rename to x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleRuntimeFieldHelper.java index b8be31179a51e..4e825db4b3c57 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleRuntimeValues.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleRuntimeFieldHelper.java @@ -23,7 +23,7 @@ /** * Manages the creation of doc values and queries for {@code double} fields. */ -public final class DoubleRuntimeValues extends AbstractRuntimeValues { +public final class DoubleRuntimeFieldHelper { @FunctionalInterface public interface NewLeafLoader { IntConsumer leafLoader(LeafReaderContext ctx, DoubleConsumer sync) throws IOException; @@ -31,36 +31,31 @@ public interface NewLeafLoader { private final NewLeafLoader newLeafLoader; - public DoubleRuntimeValues(NewLeafLoader newLeafLoader) { + public DoubleRuntimeFieldHelper(NewLeafLoader newLeafLoader) { this.newLeafLoader = newLeafLoader; } public CheckedFunction docValues() { - return unstarted().docValues(); + return new Values().docValues(); } public Query existsQuery(String fieldName) { - return unstarted().new ExistsQuery(fieldName); + return new Values().new ExistsQuery(fieldName); } public Query rangeQuery(String fieldName, double lowerValue, double upperValue) { - return unstarted().new RangeQuery(fieldName, lowerValue, upperValue); + return new Values().new RangeQuery(fieldName, lowerValue, upperValue); } public Query termQuery(String fieldName, double value) { - return unstarted().new TermQuery(fieldName, value); + return new Values().new TermQuery(fieldName, value); } public Query termsQuery(String fieldName, double... value) { - return unstarted().new TermsQuery(fieldName, value); + return new Values().new TermsQuery(fieldName, value); } - @Override - protected SharedValues newSharedValues() { - return new SharedValues(); - } - - protected class SharedValues extends AbstractRuntimeValues.SharedValues { + private class Values extends AbstractRuntimeValues { private double[] values = new double[1]; @Override diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java index c5aadc82a4c3b..14e0ae8e0fd82 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java @@ -48,11 +48,11 @@ public interface LeafFactory { DoubleScriptFieldScript newInstance(LeafReaderContext ctx, DoubleConsumer sync) throws IOException; /** - * Build an {@link DoubleRuntimeValues} to manage creation of doc + * Build an {@link DoubleRuntimeFieldHelper} to manage creation of doc * values and queries using the script. */ - default DoubleRuntimeValues runtimeValues() { - return new DoubleRuntimeValues((ctx, sync) -> { + default DoubleRuntimeFieldHelper runtimeValues() { + return new DoubleRuntimeFieldHelper((ctx, sync) -> { DoubleScriptFieldScript script = newInstance(ctx, sync); return docId -> { script.setDocId(docId); diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongRuntimeValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongRuntimeFieldHelper.java similarity index 92% rename from x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongRuntimeValues.java rename to x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongRuntimeFieldHelper.java index 9a18e3fdd9faf..d52a68d294dae 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongRuntimeValues.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongRuntimeFieldHelper.java @@ -23,7 +23,7 @@ /** * Manages the creation of doc values and queries for {@code long} fields. */ -public final class LongRuntimeValues extends AbstractRuntimeValues { +public final class LongRuntimeFieldHelper { @FunctionalInterface public interface NewLeafLoader { IntConsumer leafLoader(LeafReaderContext ctx, LongConsumer sync) throws IOException; @@ -31,36 +31,31 @@ public interface NewLeafLoader { private final NewLeafLoader newLeafLoader; - public LongRuntimeValues(NewLeafLoader newLeafLoader) { + public LongRuntimeFieldHelper(NewLeafLoader newLeafLoader) { this.newLeafLoader = newLeafLoader; } public CheckedFunction docValues() { - return unstarted().docValues(); + return new Values().docValues(); } public Query existsQuery(String fieldName) { - return unstarted().new ExistsQuery(fieldName); + return new Values().new ExistsQuery(fieldName); } public Query rangeQuery(String fieldName, long lowerValue, long upperValue) { - return unstarted().new RangeQuery(fieldName, lowerValue, upperValue); + return new Values().new RangeQuery(fieldName, lowerValue, upperValue); } public Query termQuery(String fieldName, long value) { - return unstarted().new TermQuery(fieldName, value); + return new Values().new TermQuery(fieldName, value); } public Query termsQuery(String fieldName, long... value) { - return unstarted().new TermsQuery(fieldName, value); + return new Values().new TermsQuery(fieldName, value); } - @Override - protected SharedValues newSharedValues() { - return new SharedValues(); - } - - protected class SharedValues extends AbstractRuntimeValues.SharedValues { + private class Values extends AbstractRuntimeValues { private long[] values = new long[1]; @Override diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java index 0325d7b3cb744..6a52e28968458 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java @@ -48,11 +48,11 @@ public interface LeafFactory { LongScriptFieldScript newInstance(LeafReaderContext ctx, LongConsumer sync) throws IOException; /** - * Build an {@link LongRuntimeValues} to manage creation of doc + * Build an {@link LongRuntimeFieldHelper} to manage creation of doc * values and queries using the script. */ - default LongRuntimeValues runtimeValues() { - return new LongRuntimeValues((ctx, sync) -> { + default LongRuntimeFieldHelper runtimeValues() { + return new LongRuntimeFieldHelper((ctx, sync) -> { LongScriptFieldScript script = newInstance(ctx, sync); return docId -> { script.setDocId(docId); diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeFieldHelper.java similarity index 93% rename from x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java rename to x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeFieldHelper.java index b4e490a63256d..ba3375ad9de70 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeValues.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringRuntimeFieldHelper.java @@ -30,7 +30,7 @@ /** * Manages the creation of doc values and queries for {@link String} fields. */ -public final class StringRuntimeValues extends AbstractRuntimeValues { +public final class StringRuntimeFieldHelper { @FunctionalInterface public interface NewLeafLoader { IntConsumer leafLoader(LeafReaderContext ctx, Consumer sync) throws IOException; @@ -38,52 +38,47 @@ public interface NewLeafLoader { private final NewLeafLoader newLeafLoader; - public StringRuntimeValues(NewLeafLoader newLeafLoader) { + public StringRuntimeFieldHelper(NewLeafLoader newLeafLoader) { this.newLeafLoader = newLeafLoader; } public CheckedFunction docValues() { - return unstarted().docValues(); + return new Values().docValues(); } public Query existsQuery(String fieldName) { - return unstarted().new ExistsQuery(fieldName); + return new Values().new ExistsQuery(fieldName); } public Query fuzzyQuery(String fieldName, String value, int maxEdits, int prefixLength, int maxExpansions, boolean transpositions) { - return unstarted().new FuzzyQuery(fieldName, value, maxEdits, prefixLength, maxExpansions, transpositions); + return new Values().new FuzzyQuery(fieldName, value, maxEdits, prefixLength, maxExpansions, transpositions); } public Query prefixQuery(String fieldName, String value) { - return unstarted().new PrefixQuery(fieldName, value); + return new Values().new PrefixQuery(fieldName, value); } public Query rangeQuery(String fieldName, String lowerValue, String upperValue, boolean includeLower, boolean includeUpper) { - return unstarted().new RangeQuery(fieldName, lowerValue, upperValue, includeLower, includeUpper); + return new Values().new RangeQuery(fieldName, lowerValue, upperValue, includeLower, includeUpper); } public Query regexpQuery(String fieldName, String pattern, int flags, int maxDeterminizedStates) { - return unstarted().new RegexpQuery(fieldName, pattern, flags, maxDeterminizedStates); + return new Values().new RegexpQuery(fieldName, pattern, flags, maxDeterminizedStates); } public Query termQuery(String fieldName, String value) { - return unstarted().new TermQuery(fieldName, value); + return new Values().new TermQuery(fieldName, value); } public Query termsQuery(String fieldName, String... values) { - return unstarted().new TermsQuery(fieldName, values); + return new Values().new TermsQuery(fieldName, values); } public Query wildcardQuery(String fieldName, String pattern) { - return unstarted().new WildcardQuery(fieldName, pattern); + return new Values().new WildcardQuery(fieldName, pattern); } - @Override - protected SharedValues newSharedValues() { - return new SharedValues(); - } - - protected class SharedValues extends AbstractRuntimeValues.SharedValues { + private class Values extends AbstractRuntimeValues { private String[] values = new String[1]; @Override diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java index 5d63073fc4cd2..505fde508f312 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java @@ -51,11 +51,11 @@ public interface LeafFactory { StringScriptFieldScript newInstance(LeafReaderContext ctx, Consumer sync) throws IOException; /** - * Build an {@link StringRuntimeValues} to manage creation of doc + * Build an {@link StringRuntimeFieldHelper} to manage creation of doc * values and queries using the script. */ - default StringRuntimeValues runtimeValues() { - return new StringRuntimeValues((ctx, sync) -> { + default StringRuntimeFieldHelper runtimeValues() { + return new StringRuntimeFieldHelper((ctx, sync) -> { StringScriptFieldScript script = newInstance(ctx, sync); return docId -> { script.setDocId(docId); diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldType.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldType.java index 996acb71bd02e..bb98e6ed976c1 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldType.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldType.java @@ -20,7 +20,7 @@ import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.script.Script; -import org.elasticsearch.xpack.runtimefields.StringRuntimeValues; +import org.elasticsearch.xpack.runtimefields.StringRuntimeFieldHelper; import org.elasticsearch.xpack.runtimefields.StringScriptFieldScript; import org.elasticsearch.xpack.runtimefields.fielddata.ScriptBinaryFieldData; @@ -68,7 +68,7 @@ public String typeName() { return ScriptFieldMapper.CONTENT_TYPE; } - private StringRuntimeValues runtimeValues(QueryShardContext ctx) { + private StringRuntimeFieldHelper runtimeValues(QueryShardContext ctx) { // TODO cache the runtimeValues in the context somehow return scriptFactory.newFactory(script.getParams(), ctx.lookup()).runtimeValues(); } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java index 44dc32a793a34..0ec948f9fe49c 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java @@ -38,7 +38,7 @@ public class DoubleScriptFieldScriptTests extends ScriptFieldScriptTestCase< DoubleScriptFieldScript.Factory, - DoubleRuntimeValues, + DoubleRuntimeFieldHelper, SortedNumericDoubleValues, Double> { @@ -74,13 +74,13 @@ public void testMultipleDocValuesValues() throws IOException { public void testExistsQuery() throws IOException { TestCase c = multipleValuesInDocValues(); - DoubleRuntimeValues isOnePointOne = c.testScript("is_one_point_one"); + DoubleRuntimeFieldHelper isOnePointOne = c.testScript("is_one_point_one"); assertThat(c.collect(isOnePointOne.existsQuery("foo"), isOnePointOne), equalTo(List.of(1.1))); } public void testTermQuery() throws IOException { TestCase c = multipleValuesInDocValues(); - DoubleRuntimeValues timesTen = c.testScript("times_nine_point_nine"); + DoubleRuntimeFieldHelper timesTen = c.testScript("times_nine_point_nine"); assertThat(c.collect(timesTen.termQuery("foo", 1), timesTen), equalTo(List.of())); assertThat(c.collect(timesTen.termQuery("foo", 10.89), timesTen), equalTo(List.of(10.89, 21.78))); assertThat(c.collect(timesTen.termQuery("foo", 21.78), timesTen), equalTo(List.of(10.89, 21.78))); @@ -89,7 +89,7 @@ public void testTermQuery() throws IOException { public void testTermsQuery() throws IOException { TestCase c = multipleValuesInDocValues(); - DoubleRuntimeValues timesTen = c.testScript("times_nine_point_nine"); + DoubleRuntimeFieldHelper timesTen = c.testScript("times_nine_point_nine"); assertThat(c.collect(timesTen.termsQuery("foo", 1, 2), timesTen), equalTo(List.of())); assertThat(c.collect(timesTen.termsQuery("foo", 10.89, 11), timesTen), equalTo(List.of(10.89, 21.78))); assertThat(c.collect(timesTen.termsQuery("foo", 21.78, 22), timesTen), equalTo(List.of(10.89, 21.78))); @@ -99,7 +99,7 @@ public void testTermsQuery() throws IOException { public void testRangeQuery() throws IOException { TestCase c = multipleValuesInDocValues(); - DoubleRuntimeValues timesTen = c.testScript("times_nine_point_nine"); + DoubleRuntimeFieldHelper timesTen = c.testScript("times_nine_point_nine"); assertThat(c.collect(timesTen.rangeQuery("foo", 1, 2), timesTen), equalTo(List.of())); assertThat(c.collect(timesTen.rangeQuery("foo", 9, 11), timesTen), equalTo(List.of(10.89, 21.78))); assertThat(c.collect(timesTen.rangeQuery("foo", 10.89, 11), timesTen), equalTo(List.of(10.89, 21.78))); @@ -158,12 +158,13 @@ protected ScriptContext scriptContext() { } @Override - protected DoubleRuntimeValues newValues(Factory factory, Map params, SearchLookup searchLookup) throws IOException { + protected DoubleRuntimeFieldHelper newHelper(Factory factory, Map params, SearchLookup searchLookup) + throws IOException { return factory.newFactory(params, searchLookup).runtimeValues(); } @Override - protected CheckedFunction docValuesBuilder(DoubleRuntimeValues values) { + protected CheckedFunction docValuesBuilder(DoubleRuntimeFieldHelper values) { return values.docValues(); } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java index 6ae25e6d77490..a1b6b64ee037b 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java @@ -38,7 +38,7 @@ public class LongScriptFieldScriptTests extends ScriptFieldScriptTestCase< LongScriptFieldScript.Factory, - LongRuntimeValues, + LongRuntimeFieldHelper, SortedNumericDocValues, Long> { @@ -69,13 +69,13 @@ public void testMultipleDocValuesValues() throws IOException { public void testExistsQuery() throws IOException { TestCase c = multipleValuesInDocValues(); - LongRuntimeValues isOne = c.testScript("is_one"); + LongRuntimeFieldHelper isOne = c.testScript("is_one"); assertThat(c.collect(isOne.existsQuery("foo"), isOne), equalTo(List.of(1L))); } public void testTermQuery() throws IOException { TestCase c = multipleValuesInDocValues(); - LongRuntimeValues timesTen = c.testScript("times_ten"); + LongRuntimeFieldHelper timesTen = c.testScript("times_ten"); assertThat(c.collect(timesTen.termQuery("foo", 1), timesTen), equalTo(List.of())); assertThat(c.collect(timesTen.termQuery("foo", 10), timesTen), equalTo(List.of(10L, 20L))); assertThat(c.collect(timesTen.termQuery("foo", 20), timesTen), equalTo(List.of(10L, 20L))); @@ -84,7 +84,7 @@ public void testTermQuery() throws IOException { public void testTermsQuery() throws IOException { TestCase c = multipleValuesInDocValues(); - LongRuntimeValues timesTen = c.testScript("times_ten"); + LongRuntimeFieldHelper timesTen = c.testScript("times_ten"); assertThat(c.collect(timesTen.termsQuery("foo", 1, 2), timesTen), equalTo(List.of())); assertThat(c.collect(timesTen.termsQuery("foo", 10, 11), timesTen), equalTo(List.of(10L, 20L))); assertThat(c.collect(timesTen.termsQuery("foo", 20, 21), timesTen), equalTo(List.of(10L, 20L))); @@ -94,7 +94,7 @@ public void testTermsQuery() throws IOException { public void testRangeQuery() throws IOException { TestCase c = multipleValuesInDocValues(); - LongRuntimeValues timesTen = c.testScript("times_ten"); + LongRuntimeFieldHelper timesTen = c.testScript("times_ten"); assertThat(c.collect(timesTen.rangeQuery("foo", 1, 2), timesTen), equalTo(List.of())); assertThat(c.collect(timesTen.rangeQuery("foo", 9, 11), timesTen), equalTo(List.of(10L, 20L))); assertThat(c.collect(timesTen.rangeQuery("foo", 10, 11), timesTen), equalTo(List.of(10L, 20L))); @@ -111,7 +111,7 @@ public void testInsideBoolTermQuery() throws IOException { * skip some of them eventually, when we're more comfortable with this. */ TestCase c = multipleValuesInDocValues(); - LongRuntimeValues timesTen = c.testScript("times_ten"); + LongRuntimeFieldHelper timesTen = c.testScript("times_ten"); assertThat( c.collect( new ForceNoBulkScoringQuery( @@ -172,12 +172,12 @@ protected ScriptContext scriptContext() { } @Override - protected LongRuntimeValues newValues(Factory factory, Map params, SearchLookup searchLookup) throws IOException { + protected LongRuntimeFieldHelper newHelper(Factory factory, Map params, SearchLookup searchLookup) throws IOException { return factory.newFactory(params, searchLookup).runtimeValues(); } @Override - protected CheckedFunction docValuesBuilder(LongRuntimeValues values) { + protected CheckedFunction docValuesBuilder(LongRuntimeFieldHelper values) { return values.docValues(); } diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java index 6d240fa46a9d4..5bc869a36690e 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/ScriptFieldScriptTestCase.java @@ -52,7 +52,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -public abstract class ScriptFieldScriptTestCase extends ESTestCase { +public abstract class ScriptFieldScriptTestCase extends ESTestCase { private final List lazyClose = new ArrayList<>(); private final ScriptService scriptService; @@ -81,9 +81,9 @@ protected List extraScriptPlugins() { protected abstract ScriptContext scriptContext(); - protected abstract V newValues(F factory, Map params, SearchLookup searchLookup) throws IOException; + protected abstract H newHelper(F factory, Map params, SearchLookup searchLookup) throws IOException; - protected abstract CheckedFunction docValuesBuilder(V values); + protected abstract CheckedFunction docValuesBuilder(H values); protected abstract void readAllDocValues(DV docValues, int docId, Consumer sync) throws IOException; @@ -113,27 +113,27 @@ private TestCase(CheckedConsumer indexBuilder) t } } - protected V script(String script) throws IOException { + protected H script(String script) throws IOException { return script(new Script(script)); } - protected V testScript(String name) throws IOException { + protected H testScript(String name) throws IOException { return script(new Script(ScriptType.INLINE, "test", name, Map.of())); } - private V script(Script script) throws IOException { - return newValues(scriptService.compile(script, scriptContext()), Map.of(), searchLookup); + private H script(Script script) throws IOException { + return newHelper(scriptService.compile(script, scriptContext()), Map.of(), searchLookup); } protected List collect(String script) throws IOException { return collect(new MatchAllDocsQuery(), script(script)); } - protected List collect(V values) throws IOException { + protected List collect(H values) throws IOException { return collect(new MatchAllDocsQuery(), values); } - protected List collect(Query query, V values) throws IOException { + protected List collect(Query query, H values) throws IOException { // Now run the query and collect the results List result = new ArrayList<>(); CheckedFunction docValuesBuilder = docValuesBuilder(values); diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java index f8b9934334951..94ccb77cf6b9b 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java @@ -37,7 +37,7 @@ public class StringScriptFieldScriptTests extends ScriptFieldScriptTestCase< StringScriptFieldScript.Factory, - StringRuntimeValues, + StringRuntimeFieldHelper, SortedBinaryDocValues, String> { @@ -70,13 +70,13 @@ public void testMultipleDocValuesValues() throws IOException { public void testExistsQuery() throws IOException { TestCase c = multipleValuesInDocValues(); - StringRuntimeValues isCat = c.testScript("is_cat"); + StringRuntimeFieldHelper isCat = c.testScript("is_cat"); assertThat(c.collect(isCat.existsQuery("foo"), isCat), equalTo(List.of("cat"))); } public void testFuzzyQuery() throws IOException { TestCase c = multipleValuesInDocValues(); - StringRuntimeValues addO = c.testScript("add_o"); + StringRuntimeFieldHelper addO = c.testScript("add_o"); assertThat(c.collect(addO.fuzzyQuery("foo", "caaaaat", 1, 1, 1, true), addO), equalTo(List.of())); assertThat(c.collect(addO.fuzzyQuery("foo", "cat", 1, 1, 1, true), addO), equalTo(List.of("cato", "pigo"))); assertThat(c.collect(addO.fuzzyQuery("foo", "pig", 1, 1, 1, true), addO), equalTo(List.of("cato", "pigo"))); @@ -85,7 +85,7 @@ public void testFuzzyQuery() throws IOException { public void testTermQuery() throws IOException { TestCase c = multipleValuesInDocValues(); - StringRuntimeValues addO = c.testScript("add_o"); + StringRuntimeFieldHelper addO = c.testScript("add_o"); assertThat(c.collect(addO.termQuery("foo", "cat"), addO), equalTo(List.of())); assertThat(c.collect(addO.termQuery("foo", "cato"), addO), equalTo(List.of("cato", "pigo"))); assertThat(c.collect(addO.termQuery("foo", "pigo"), addO), equalTo(List.of("cato", "pigo"))); @@ -94,7 +94,7 @@ public void testTermQuery() throws IOException { public void testTermsQuery() throws IOException { TestCase c = multipleValuesInDocValues(); - StringRuntimeValues addO = c.testScript("add_o"); + StringRuntimeFieldHelper addO = c.testScript("add_o"); assertThat(c.collect(addO.termsQuery("foo", "cat", "dog"), addO), equalTo(List.of())); assertThat(c.collect(addO.termsQuery("foo", "cato", "piglet"), addO), equalTo(List.of("cato", "pigo"))); assertThat(c.collect(addO.termsQuery("foo", "pigo", "catington"), addO), equalTo(List.of("cato", "pigo"))); @@ -103,7 +103,7 @@ public void testTermsQuery() throws IOException { public void testPrefixQuery() throws IOException { TestCase c = multipleValuesInDocValues(); - StringRuntimeValues addO = c.testScript("add_o"); + StringRuntimeFieldHelper addO = c.testScript("add_o"); assertThat(c.collect(addO.prefixQuery("foo", "catdog"), addO), equalTo(List.of())); assertThat(c.collect(addO.prefixQuery("foo", "cat"), addO), equalTo(List.of("cato", "pigo"))); assertThat(c.collect(addO.prefixQuery("foo", "pig"), addO), equalTo(List.of("cato", "pigo"))); @@ -113,7 +113,7 @@ public void testPrefixQuery() throws IOException { public void testRangeQuery() throws IOException { TestCase c = multipleValuesInDocValues(); - StringRuntimeValues addO = c.testScript("add_o"); + StringRuntimeFieldHelper addO = c.testScript("add_o"); assertThat(c.collect(addO.rangeQuery("foo", "catz", "cbat", false, false), addO), equalTo(List.of())); assertThat(c.collect(addO.rangeQuery("foo", "c", "cb", false, false), addO), equalTo(List.of("cato", "pigo"))); assertThat(c.collect(addO.rangeQuery("foo", "p", "q", false, false), addO), equalTo(List.of("cato", "pigo"))); @@ -126,7 +126,7 @@ public void testRangeQuery() throws IOException { public void testRegexpQuery() throws IOException { TestCase c = multipleValuesInDocValues(); - StringRuntimeValues addO = c.testScript("add_o"); + StringRuntimeFieldHelper addO = c.testScript("add_o"); assertThat(c.collect(addO.regexpQuery("foo", "cat", RegExp.ALL, 100000), addO), equalTo(List.of())); assertThat(c.collect(addO.regexpQuery("foo", "cat[aeiou]", RegExp.ALL, 100000), addO), equalTo(List.of("cato", "pigo"))); assertThat(c.collect(addO.regexpQuery("foo", "p.*", RegExp.ALL, 100000), addO), equalTo(List.of("cato", "pigo"))); @@ -135,7 +135,7 @@ public void testRegexpQuery() throws IOException { public void testWildcardQuery() throws IOException { TestCase c = multipleValuesInDocValues(); - StringRuntimeValues addO = c.testScript("add_o"); + StringRuntimeFieldHelper addO = c.testScript("add_o"); assertThat(c.collect(addO.wildcardQuery("foo", "cat"), addO), equalTo(List.of())); assertThat(c.collect(addO.wildcardQuery("foo", "cat?"), addO), equalTo(List.of("cato", "pigo"))); assertThat(c.collect(addO.wildcardQuery("foo", "p*"), addO), equalTo(List.of("cato", "pigo"))); @@ -263,12 +263,13 @@ protected ScriptContext scriptContext() { } @Override - protected StringRuntimeValues newValues(Factory factory, Map params, SearchLookup searchLookup) throws IOException { + protected StringRuntimeFieldHelper newHelper(Factory factory, Map params, SearchLookup searchLookup) + throws IOException { return factory.newFactory(params, searchLookup).runtimeValues(); } @Override - protected CheckedFunction docValuesBuilder(StringRuntimeValues values) { + protected CheckedFunction docValuesBuilder(StringRuntimeFieldHelper values) { return values.docValues(); } From 724aabae0c7dd4e5f866d6195a8b3afa85cae2c9 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 8 Jul 2020 12:49:49 -0400 Subject: [PATCH 34/35] Rename --- .../DoubleScriptFieldScript.java | 2 +- .../runtimefields/LongScriptFieldScript.java | 2 +- .../StringScriptFieldScript.java | 2 +- .../fielddata/ScriptBinaryFieldData.java | 2 +- .../mapper/RuntimeKeywordMappedFieldType.java | 29 +++++++------------ .../DoubleScriptFieldScriptTests.java | 2 +- .../LongScriptFieldScriptTests.java | 2 +- .../StringScriptFieldScriptTests.java | 2 +- 8 files changed, 18 insertions(+), 25 deletions(-) diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java index 14e0ae8e0fd82..61d5364a30e6b 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScript.java @@ -51,7 +51,7 @@ public interface LeafFactory { * Build an {@link DoubleRuntimeFieldHelper} to manage creation of doc * values and queries using the script. */ - default DoubleRuntimeFieldHelper runtimeValues() { + default DoubleRuntimeFieldHelper runtimeFieldHelper() { return new DoubleRuntimeFieldHelper((ctx, sync) -> { DoubleScriptFieldScript script = newInstance(ctx, sync); return docId -> { diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java index 6a52e28968458..6a316b71be2c4 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScript.java @@ -51,7 +51,7 @@ public interface LeafFactory { * Build an {@link LongRuntimeFieldHelper} to manage creation of doc * values and queries using the script. */ - default LongRuntimeFieldHelper runtimeValues() { + default LongRuntimeFieldHelper runtimeFieldHelper() { return new LongRuntimeFieldHelper((ctx, sync) -> { LongScriptFieldScript script = newInstance(ctx, sync); return docId -> { diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java index 505fde508f312..a93d4b70060a1 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScript.java @@ -54,7 +54,7 @@ public interface LeafFactory { * Build an {@link StringRuntimeFieldHelper} to manage creation of doc * values and queries using the script. */ - default StringRuntimeFieldHelper runtimeValues() { + default StringRuntimeFieldHelper runtimeFieldHelper() { return new StringRuntimeFieldHelper((ctx, sync) -> { StringScriptFieldScript script = newInstance(ctx, sync); return docId -> { diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryFieldData.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryFieldData.java index bb76003595bcc..10999c17de694 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryFieldData.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryFieldData.java @@ -75,7 +75,7 @@ private ScriptBinaryFieldData(IndexSettings indexSettings, String fieldName, Str public void setSearchLookup(SearchLookup searchLookup) { // TODO wire the params from the mappings definition, we don't parse them yet // TODO it'd be nice if we could stuff `runtimeValues` some place into the search context so we could reuse it - this.docValuesBuilder.set(scriptFactory.newFactory(Collections.emptyMap(), searchLookup).runtimeValues().docValues()); + this.docValuesBuilder.set(scriptFactory.newFactory(Collections.emptyMap(), searchLookup).runtimeFieldHelper().docValues()); } @Override diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldType.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldType.java index bb98e6ed976c1..0a5eceef35b4c 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldType.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeKeywordMappedFieldType.java @@ -6,8 +6,8 @@ package org.elasticsearch.xpack.runtimefields.mapper; -import org.apache.lucene.search.Query; import org.apache.lucene.search.MultiTermQuery.RewriteMethod; +import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.geo.ShapeRelation; import org.elasticsearch.common.lucene.BytesRefs; @@ -68,9 +68,8 @@ public String typeName() { return ScriptFieldMapper.CONTENT_TYPE; } - private StringRuntimeFieldHelper runtimeValues(QueryShardContext ctx) { - // TODO cache the runtimeValues in the context somehow - return scriptFactory.newFactory(script.getParams(), ctx.lookup()).runtimeValues(); + private StringRuntimeFieldHelper helper(QueryShardContext ctx) { + return scriptFactory.newFactory(script.getParams(), ctx.lookup()).runtimeFieldHelper(); } @Override @@ -81,7 +80,7 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName) { @Override public Query existsQuery(QueryShardContext context) { - return runtimeValues(context).existsQuery(name()); + return helper(context).existsQuery(name()); } @Override @@ -94,12 +93,12 @@ public Query fuzzyQuery( QueryShardContext context ) { String term = BytesRefs.toString(value); - return runtimeValues(context).fuzzyQuery(name(), term, fuzziness.asDistance(term), prefixLength, maxExpansions, transpositions); + return helper(context).fuzzyQuery(name(), term, fuzziness.asDistance(term), prefixLength, maxExpansions, transpositions); } @Override public Query prefixQuery(String value, RewriteMethod method, QueryShardContext context) { - return runtimeValues(context).prefixQuery(name(), value); + return helper(context).prefixQuery(name(), value); } @Override @@ -113,33 +112,27 @@ public Query rangeQuery( DateMathParser parser, QueryShardContext context ) { - return runtimeValues(context).rangeQuery( - name(), - BytesRefs.toString(lowerTerm), - BytesRefs.toString(upperTerm), - includeLower, - includeUpper - ); + return helper(context).rangeQuery(name(), BytesRefs.toString(lowerTerm), BytesRefs.toString(upperTerm), includeLower, includeUpper); } @Override public Query regexpQuery(String value, int flags, int maxDeterminizedStates, RewriteMethod method, QueryShardContext context) { - return runtimeValues(context).regexpQuery(name(), value, flags, maxDeterminizedStates); + return helper(context).regexpQuery(name(), value, flags, maxDeterminizedStates); } @Override public Query termQuery(Object value, QueryShardContext context) { - return runtimeValues(context).termQuery(name(), BytesRefs.toString(value)); + return helper(context).termQuery(name(), BytesRefs.toString(value)); } @Override public Query termsQuery(List values, QueryShardContext context) { - return runtimeValues(context).termsQuery(name(), values.stream().map(BytesRefs::toString).toArray(String[]::new)); + return helper(context).termsQuery(name(), values.stream().map(BytesRefs::toString).toArray(String[]::new)); } @Override public Query wildcardQuery(String value, RewriteMethod method, QueryShardContext context) { - return runtimeValues(context).wildcardQuery(name(), value); + return helper(context).wildcardQuery(name(), value); } void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java index 0ec948f9fe49c..e47ce41040f11 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/DoubleScriptFieldScriptTests.java @@ -160,7 +160,7 @@ protected ScriptContext scriptContext() { @Override protected DoubleRuntimeFieldHelper newHelper(Factory factory, Map params, SearchLookup searchLookup) throws IOException { - return factory.newFactory(params, searchLookup).runtimeValues(); + return factory.newFactory(params, searchLookup).runtimeFieldHelper(); } @Override diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java index a1b6b64ee037b..4fa4472c37c9f 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/LongScriptFieldScriptTests.java @@ -173,7 +173,7 @@ protected ScriptContext scriptContext() { @Override protected LongRuntimeFieldHelper newHelper(Factory factory, Map params, SearchLookup searchLookup) throws IOException { - return factory.newFactory(params, searchLookup).runtimeValues(); + return factory.newFactory(params, searchLookup).runtimeFieldHelper(); } @Override diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java index 94ccb77cf6b9b..6caaf2ae9b3f4 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/StringScriptFieldScriptTests.java @@ -265,7 +265,7 @@ protected ScriptContext scriptContext() { @Override protected StringRuntimeFieldHelper newHelper(Factory factory, Map params, SearchLookup searchLookup) throws IOException { - return factory.newFactory(params, searchLookup).runtimeValues(); + return factory.newFactory(params, searchLookup).runtimeFieldHelper(); } @Override From 6645c09480234ca9dabf15d7155075f8056c7c36 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 8 Jul 2020 12:55:09 -0400 Subject: [PATCH 35/35] plumb --- .../index/query/QueryShardContext.java | 3 +-- .../elasticsearch/search/lookup/DocLookup.java | 6 ++---- .../search/lookup/LeafDocLookup.java | 18 ++---------------- .../search/lookup/SearchLookup.java | 2 +- .../search/lookup/LeafDocLookupTests.java | 4 +--- 5 files changed, 7 insertions(+), 26 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java index b550c6b5552b2..be449f625c399 100644 --- a/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java +++ b/server/src/main/java/org/elasticsearch/index/query/QueryShardContext.java @@ -297,8 +297,7 @@ MappedFieldType failIfFieldMappingNotFound(String name, MappedFieldType fieldMap public SearchLookup lookup() { if (lookup == null) { - lookup = new SearchLookup(getMapperService(), - mappedFieldType -> indexFieldDataService.apply(mappedFieldType, fullyQualifiedIndex.getName())); + lookup = new SearchLookup(getMapperService(), this::getForField); } return lookup; } diff --git a/server/src/main/java/org/elasticsearch/search/lookup/DocLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/DocLookup.java index 7fd6a21adc8ec..1f0e53fca14c9 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/DocLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/DocLookup.java @@ -29,12 +29,10 @@ public class DocLookup { private final MapperService mapperService; private final Function> fieldDataLookup; - private final SearchLookup searchLookup; - public DocLookup(MapperService mapperService, Function> fieldDataLookup, SearchLookup searchLookup) { + public DocLookup(MapperService mapperService, Function> fieldDataLookup) { this.mapperService = mapperService; this.fieldDataLookup = fieldDataLookup; - this.searchLookup = searchLookup; } public MapperService mapperService() { @@ -46,6 +44,6 @@ public IndexFieldData getForField(MappedFieldType fieldType) { } public LeafDocLookup getLeafDocLookup(LeafReaderContext context) { - return new LeafDocLookup(mapperService, fieldDataLookup, searchLookup, context); + return new LeafDocLookup(mapperService, fieldDataLookup, context); } } diff --git a/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java index 791ae06e1a3c2..1098f979cc285 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/LeafDocLookup.java @@ -21,9 +21,7 @@ import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.LeafFieldData; import org.elasticsearch.index.fielddata.ScriptDocValues; -import org.elasticsearch.index.fielddata.SearchLookupAware; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperService; @@ -43,20 +41,13 @@ public class LeafDocLookup implements Map> { private final MapperService mapperService; private final Function> fieldDataLookup; - private final SearchLookup searchLookup; private final LeafReaderContext reader; private int docId = -1; - LeafDocLookup( - MapperService mapperService, - Function> fieldDataLookup, - SearchLookup searchLookup, - LeafReaderContext reader - ) { + LeafDocLookup(MapperService mapperService, Function> fieldDataLookup, LeafReaderContext reader) { this.mapperService = mapperService; this.fieldDataLookup = fieldDataLookup; - this.searchLookup = searchLookup; this.reader = reader; } @@ -83,12 +74,7 @@ public ScriptDocValues get(Object key) { scriptValues = AccessController.doPrivileged(new PrivilegedAction>() { @Override public ScriptDocValues run() { - // TODO should this go through QueryShardContext? - IndexFieldData ifd = fieldDataLookup.apply(fieldType); - if (ifd instanceof SearchLookupAware) { - ((SearchLookupAware) ifd).setSearchLookup(searchLookup); - } - return ifd.load(reader).getScriptValues(); + return fieldDataLookup.apply(fieldType).load(reader).getScriptValues(); } }); localCacheFieldData.put(fieldName, scriptValues); diff --git a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java index b723406c31ae7..04aef7d2e8f63 100644 --- a/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java +++ b/server/src/main/java/org/elasticsearch/search/lookup/SearchLookup.java @@ -35,7 +35,7 @@ public class SearchLookup { final FieldsLookup fieldsLookup; public SearchLookup(MapperService mapperService, Function> fieldDataLookup) { - docMap = new DocLookup(mapperService, fieldDataLookup, this); + docMap = new DocLookup(mapperService, fieldDataLookup); sourceLookup = new SourceLookup(); fieldsLookup = new FieldsLookup(mapperService); } diff --git a/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java b/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java index 8dbc73767fbee..0df1553d319dc 100644 --- a/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java +++ b/server/src/test/java/org/elasticsearch/search/lookup/LeafDocLookupTests.java @@ -51,9 +51,7 @@ public void setUp() throws Exception { docValues = mock(ScriptDocValues.class); IndexFieldData fieldData = createFieldData(docValues); - docLookup = new LeafDocLookup(mapperService, - ignored -> fieldData, - null, null); + docLookup = new LeafDocLookup(mapperService, ignored -> fieldData, null); } public void testBasicLookup() {