Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Some queries for runtime fields #58940

Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
0cfdf33
Add runtime fields plugin under x-pack
javanna Jun 17, 2020
e474582
Hack together some script contexts
nik9000 Jun 17, 2020
76d8b94
Scripted fields: rework script contexts (#58342)
nik9000 Jun 18, 2020
818403b
Spotless and maybe tests?
nik9000 Jun 18, 2020
3669ec4
Fixup test case after merge
nik9000 Jun 26, 2020
3a390b5
Get runtime fields tests passing
nik9000 Jun 26, 2020
59b2a18
Add queries
nik9000 Jun 26, 2020
28944a4
Better way for queries
nik9000 Jun 29, 2020
7d43642
Fails
nik9000 Jun 29, 2020
89d6352
Shared iteration
nik9000 Jun 30, 2020
5b353ce
Long term query
nik9000 Jun 30, 2020
23af466
Common stuffs?
nik9000 Jun 30, 2020
acf64c8
Double queries
nik9000 Jun 30, 2020
7ee42c0
Rename
nik9000 Jun 30, 2020
216c02c
Range query
nik9000 Jun 30, 2020
09e6feb
Fuzzy
nik9000 Jun 30, 2020
4e66638
Wildcard
nik9000 Jun 30, 2020
6837172
Regexp query
nik9000 Jun 30, 2020
f5106b2
Exists and some sorting
nik9000 Jun 30, 2020
2e05521
Spotless
nik9000 Jun 30, 2020
bfe15f8
WIP
nik9000 Jun 30, 2020
7c18ddb
Fixup bool
nik9000 Jul 1, 2020
79d41c8
Remove some duplication
nik9000 Jul 2, 2020
e3850a7
Merge branch 'feature/runtime_fields' into runtime_field_string_term_…
nik9000 Jul 2, 2020
ab6834e
Javadoc
nik9000 Jul 2, 2020
4e83233
Javadoc
nik9000 Jul 2, 2020
810568b
Javadoc
nik9000 Jul 2, 2020
3b79704
Merge branch 'feature/runtime_fields' into runtime_field_string_term_…
nik9000 Jul 2, 2020
bc80a5e
Wire in doc values and term query
nik9000 Jul 2, 2020
93a031f
Integrate!
nik9000 Jul 2, 2020
3409621
Its over nine thousand!
nik9000 Jul 2, 2020
b5b02ed
In a bool
nik9000 Jul 2, 2020
e17fef8
itr
nik9000 Jul 6, 2020
edc571f
Drop shared iteration tests
nik9000 Jul 8, 2020
f90a0c7
Remove caching
nik9000 Jul 8, 2020
724aaba
Rename
nik9000 Jul 8, 2020
6645c09
plumb
nik9000 Jul 8, 2020
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -297,8 +297,7 @@ MappedFieldType failIfFieldMappingNotFound(String name, MappedFieldType fieldMap

public SearchLookup lookup() {
if (lookup == null) {
lookup = new SearchLookup(getMapperService(),
mappedFieldType -> indexFieldDataService.apply(mappedFieldType, fullyQualifiedIndex.getName()));
lookup = new SearchLookup(getMapperService(), this::getForField);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

add a TODO here, to remind us that we need to figure out if this should be merged upstream or not? once the searchlookupaware hack is removed and fielddatabuilder is adapted, we will not need this change anyways?

}
return lookup;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,7 @@ public class LeafDocLookup implements Map<String, ScriptDocValues<?>> {

private int docId = -1;

LeafDocLookup(MapperService mapperService, Function<MappedFieldType, IndexFieldData<?>> fieldDataLookup,
LeafReaderContext reader) {
LeafDocLookup(MapperService mapperService, Function<MappedFieldType, IndexFieldData<?>> fieldDataLookup, LeafReaderContext reader) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can you revert please? Trying to trim down changes to stuff that's outside of the runtime fields plugin

this.mapperService = mapperService;
this.fieldDataLookup = fieldDataLookup;
this.reader = reader;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,7 @@ public void setUp() throws Exception {
docValues = mock(ScriptDocValues.class);
IndexFieldData<?> fieldData = createFieldData(docValues);

docLookup = new LeafDocLookup(mapperService,
ignored -> fieldData,
null);
docLookup = new LeafDocLookup(mapperService, ignored -> fieldData, null);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

same here, let's revert?

}

public void testBasicLookup() {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/

package org.elasticsearch.xpack.runtimefields;

import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.ConstantScoreScorer;
import org.apache.lucene.search.ConstantScoreWeight;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreMode;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.TwoPhaseIterator;
import org.apache.lucene.search.Weight;

import java.io.IOException;
import java.util.Objects;
import java.util.function.IntConsumer;

/**
* Abstract base for implementing doc values and queries against values
* calculated at runtime.
*/
public abstract class AbstractRuntimeValues {
protected int count;
private boolean sort;

private int docId = -1;
private int maxDoc;

protected final IntConsumer leafCursor(LeafReaderContext ctx) throws IOException {
IntConsumer leafLoader = newLeafLoader(ctx);
docId = -1;
maxDoc = ctx.reader().maxDoc();
return new IntConsumer() {
@Override
public void accept(int targetDocId) {
if (docId == targetDocId) {
return;
}
docId = targetDocId;
count = 0;
leafLoader.accept(targetDocId);
if (sort) {
sort();
}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this looks suspiciously similar to the advance method of a doc_values impl. I think I see why it is convenient to have something that accepts the doc_id and makes the values available, I guess I find it hard to follow the indirections from NewLeafLoader to IntConsumer.

I think ideally we would make it possible to call script.getResult(docId) or something along those lines. That could be called from the existing ScriptBinaryDocValues implementation which also sorts values, or from the two phase iterator on the query side.

Do we ever need sorted values on the query side?

}
};
}

protected final void alwaysSortResults() {
sort = true;
}

protected final int docId() {
return docId;
}

protected final int maxDoc() {
return maxDoc;
}

protected abstract IntConsumer newLeafLoader(LeafReaderContext ctx) throws IOException;

protected abstract void sort();

protected abstract class AbstractRuntimeQuery extends Query {
protected final String fieldName;

protected AbstractRuntimeQuery(String fieldName) {
this.fieldName = fieldName;
}

@Override
public final Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
return new ConstantScoreWeight(this, boost) {
@Override
public boolean isCacheable(LeafReaderContext ctx) {
return false; // scripts aren't really cacheable at this point
}

@Override
public Scorer scorer(LeafReaderContext ctx) throws IOException {
IntConsumer leafCursor = leafCursor(ctx);
DocIdSetIterator approximation = DocIdSetIterator.all(ctx.reader().maxDoc());
TwoPhaseIterator twoPhase = new TwoPhaseIterator(approximation) {
@Override
public boolean matches() throws IOException {
leafCursor.accept(approximation.docID());
return AbstractRuntimeQuery.this.matches();
}

@Override
public float matchCost() {
// TODO we don't have a good way of estimating the complexity of the script so we just go with 9000
return approximation().cost() * 9000f;
}
};
return new ConstantScoreScorer(this, score(), scoreMode, twoPhase);
}
};
}

protected abstract boolean matches();

@Override
public final String toString(String field) {
if (fieldName.contentEquals(field)) {
return bareToString();
}
return fieldName + ":" + bareToString();
}

protected abstract String bareToString();

@Override
public int hashCode() {
return Objects.hash(fieldName);
}

@Override
public boolean equals(Object obj) {
if (obj == null || getClass() != obj.getClass()) {
return false;
}
AbstractRuntimeQuery other = (AbstractRuntimeQuery) obj;
return fieldName.equals(other.fieldName);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -23,16 +23,16 @@ public abstract class AbstractScriptFieldScript {
private final LeafSearchLookup leafSearchLookup;

public AbstractScriptFieldScript(Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
this.leafSearchLookup = searchLookup.getLeafSearchLookup(ctx);
leafSearchLookup = searchLookup.getLeafSearchLookup(ctx);
// TODO how do other scripts get stored fields exposed? Through asMap? I don't see any getters for them.
this.params = params;
}

/**
* Set the document to run the script against.
*/
public final void setDocument(int docId) {
this.leafSearchLookup.setDocument(docId);
public final void setDocId(int docId) {
leafSearchLookup.setDocument(docId);
}

/**
Expand Down
Loading