From f61f23f4b6d00c8919932ed4dfec067764cb2a58 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Wed, 2 Jul 2014 09:45:37 -0700 Subject: [PATCH 01/16] Start of expressions integration as a script engine. Does not run (MapperService causing errors). --- pom.xml | 6 ++ .../elasticsearch/script/ScriptModule.java | 11 ++- .../elasticsearch/script/SearchScript.java | 3 +- .../script/expression/ExpressionScript.java | 87 ++++++++++++++++++ .../ExpressionScriptCompilationException.java | 14 +++ .../ExpressionScriptEngineService.java | 89 +++++++++++++++++++ .../ExpressionScriptExecutionException.java | 12 +++ .../script/ExpressionScriptTest.java | 29 ++++++ 8 files changed, 248 insertions(+), 3 deletions(-) create mode 100644 src/main/java/org/elasticsearch/script/expression/ExpressionScript.java create mode 100644 src/main/java/org/elasticsearch/script/expression/ExpressionScriptCompilationException.java create mode 100644 src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java create mode 100644 src/main/java/org/elasticsearch/script/expression/ExpressionScriptExecutionException.java create mode 100644 src/test/java/org/elasticsearch/script/ExpressionScriptTest.java diff --git a/pom.xml b/pom.xml index 863a40ee64239..66a278c384f81 100644 --- a/pom.xml +++ b/pom.xml @@ -149,6 +149,12 @@ ${lucene.version} compile + + org.apache.lucene + lucene-expressions + ${lucene.version} + compile + com.spatial4j spatial4j diff --git a/src/main/java/org/elasticsearch/script/ScriptModule.java b/src/main/java/org/elasticsearch/script/ScriptModule.java index c0d3625a67a65..c1949a3666e84 100644 --- a/src/main/java/org/elasticsearch/script/ScriptModule.java +++ b/src/main/java/org/elasticsearch/script/ScriptModule.java @@ -27,6 +27,7 @@ import org.elasticsearch.common.inject.multibindings.Multibinder; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.script.expression.ExpressionScriptEngineService; import org.elasticsearch.script.groovy.GroovyScriptEngineService; import org.elasticsearch.script.mustache.MustacheScriptEngineService; @@ -34,7 +35,8 @@ import java.util.Map; /** - * + * An {@link org.elasticsearch.common.inject.Module} which manages {@link ScriptEngineService}s, as well + * as named script */ public class ScriptModule extends AbstractModule { @@ -92,6 +94,13 @@ protected void configure() { Loggers.getLogger(ScriptService.class, settings).debug("failed to load mustache", t); } + try { + settings.getClassLoader().loadClass("org.apache.lucene.expressions.Expression"); + multibinder.addBinding().to(ExpressionScriptEngineService.class); + } catch (Throwable t) { + Loggers.getLogger(ScriptService.class, settings).debug("failed to load lucene expressions", t); + } + for (Class scriptEngine : scriptEngines) { multibinder.addBinding().to(scriptEngine); } diff --git a/src/main/java/org/elasticsearch/script/SearchScript.java b/src/main/java/org/elasticsearch/script/SearchScript.java index a02e67e12c2e4..5181d8625252f 100644 --- a/src/main/java/org/elasticsearch/script/SearchScript.java +++ b/src/main/java/org/elasticsearch/script/SearchScript.java @@ -20,7 +20,6 @@ import org.elasticsearch.common.lucene.ReaderContextAware; import org.elasticsearch.common.lucene.ScorerAware; -import org.elasticsearch.search.SearchService; import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.lookup.SearchLookup; @@ -29,7 +28,7 @@ /** * A search script. * - * @see ExplainableSearchScript for script which can explain a score + * @see {@link ExplainableSearchScript} for script which can explain a score */ public interface SearchScript extends ExecutableScript, ReaderContextAware, ScorerAware { diff --git a/src/main/java/org/elasticsearch/script/expression/ExpressionScript.java b/src/main/java/org/elasticsearch/script/expression/ExpressionScript.java new file mode 100644 index 0000000000000..864b392411737 --- /dev/null +++ b/src/main/java/org/elasticsearch/script/expression/ExpressionScript.java @@ -0,0 +1,87 @@ +package org.elasticsearch.script.expression; + +import org.apache.lucene.expressions.Bindings; +import org.apache.lucene.expressions.Expression; +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.queries.function.FunctionValues; +import org.apache.lucene.queries.function.ValueSource; +import org.apache.lucene.search.Scorer; +import org.elasticsearch.script.SearchScript; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * + */ +class ExpressionScript implements SearchScript { + + Expression expression; + Bindings bindings; + AtomicReaderContext leaf; + int docid; + + ExpressionScript(Expression e, Bindings b) { + expression = e; + bindings = b; + } + + double evaluate() { + try { + ValueSource vs = expression.getValueSource(bindings); + FunctionValues fv = vs.getValues(new HashMap(), leaf); + return fv.doubleVal(docid); + } catch (IOException e) { + throw new ExpressionScriptExecutionException("Failed to run expression", e); + } + } + + @Override + public Object run() { return new Double(evaluate()); } + + @Override + public float runAsFloat() { return (float)evaluate();} + + @Override + public long runAsLong() { return (long)evaluate(); } + + @Override + public double runAsDouble() { return evaluate(); } + + @Override + public Object unwrap(Object value) { return value; } + + @Override + public void setNextDocId(int d) { + docid = d; + } + + @Override + public void setNextReader(AtomicReaderContext l) { + leaf = l; + } + + @Override + public void setNextSource(Map source) { + + } + + @Override + public void setNextScore(float score) { + + } + + + @Override + public void setNextVar(String name, Object value) { + + } + + + + @Override + public void setScorer(Scorer scorer) { + + } +} diff --git a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptCompilationException.java b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptCompilationException.java new file mode 100644 index 0000000000000..cd56cc410ccd3 --- /dev/null +++ b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptCompilationException.java @@ -0,0 +1,14 @@ +package org.elasticsearch.script.expression; + +import org.elasticsearch.ElasticsearchException; + +import java.text.ParseException; + +/** + * Exception representing a compilation error in an expression. + */ +public class ExpressionScriptCompilationException extends ElasticsearchException { + public ExpressionScriptCompilationException(String msg, ParseException e) { + super(msg, e); + } +} diff --git a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java new file mode 100644 index 0000000000000..604308b065653 --- /dev/null +++ b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java @@ -0,0 +1,89 @@ +package org.elasticsearch.script.expression; + +import org.apache.lucene.expressions.Expression; +import org.apache.lucene.expressions.SimpleBindings; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.component.AbstractComponent; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.ScriptEngineService; +import org.elasticsearch.script.SearchScript; +import org.elasticsearch.search.lookup.SearchLookup; + +import org.apache.lucene.expressions.js.JavascriptCompiler; + +import java.text.ParseException; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Provides the infrastructure for Lucene expressions as a scripting language for Elasticsearch. + */ +public class ExpressionScriptEngineService extends AbstractComponent implements ScriptEngineService { + + private final AtomicLong counter = new AtomicLong(); + private final ClassLoader classLoader; // TODO: should use this instead of the implicit this.getClass().getClassLoader()? + private final MapperService mapper; + + @Inject + public ExpressionScriptEngineService(Settings settings, MapperService m) { + super(settings); + classLoader = settings.getClassLoader(); + mapper = m; + } + + @Override + public String[] types() { + return new String[]{"expression"}; + } + + @Override + public String[] extensions() { + return new String[]{"expr"}; + } + + @Override + public boolean sandboxed() { + return true; + } + + @Override + public Object compile(String script) { + try { + // NOTE: validation is delayed to allow runtime vars + return JavascriptCompiler.compile(script); + } catch (ParseException e) { + throw new ExpressionScriptCompilationException("Failed to parse expression: " + script, e); + } + } + + @Override + public SearchScript search(Object compiledScript, SearchLookup lookup, @Nullable Map vars) { + Expression expr = (Expression)compiledScript; + + // TODO: fill in vars into bindings, do validation + return new ExpressionScript((Expression)compiledScript, new SimpleBindings()); + } + + @Override + public ExecutableScript executable(Object compiledScript, @Nullable Map vars) { + // cannot use expressions for updates (yet) + throw new UnsupportedOperationException(); + } + + @Override + public Object execute(Object compiledScript, Map vars) { + throw new UnsupportedOperationException(); + } + + @Override + public Object unwrap(Object value) { + return value; + } + + @Override + public void close() {} +} diff --git a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptExecutionException.java b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptExecutionException.java new file mode 100644 index 0000000000000..12c6867c4960a --- /dev/null +++ b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptExecutionException.java @@ -0,0 +1,12 @@ +package org.elasticsearch.script.expression; + +import org.elasticsearch.ElasticsearchException; + +/** + * Exception used to wrap exceptions occuring while running expressions. + */ +public class ExpressionScriptExecutionException extends ElasticsearchException { + public ExpressionScriptExecutionException(String msg, Throwable cause) { + super(msg, cause); + } +} diff --git a/src/test/java/org/elasticsearch/script/ExpressionScriptTest.java b/src/test/java/org/elasticsearch/script/ExpressionScriptTest.java new file mode 100644 index 0000000000000..f8dcc2cb2379c --- /dev/null +++ b/src/test/java/org/elasticsearch/script/ExpressionScriptTest.java @@ -0,0 +1,29 @@ +package org.elasticsearch.script; + +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.test.ElasticsearchIntegrationTest; +import org.junit.Test; + +public class ExpressionScriptTest extends ElasticsearchIntegrationTest { + + @Test + public void testBasic() { + client().prepareIndex("test", "doc", "1").setSource("foo", 5).setRefresh(true).get(); + String script = "1 + 2"; + SearchResponse resp = client().prepareSearch("test") + .setSource("{\"query\": {" + + "\"function_score\": {" + + "\"query\":{\"match_all\":{}}," + + "\"boost_mode\": \"replace\"," + + "\"script_score\": {" + + "\"script\": \"" + script +"\"," + + "\"lang\": \"expression\"" + + "}" + + "}" + + "}}").get(); + + assertEquals(3, resp.getHits().getAt(0).getScore(), 0.0001f); + } + + +} From a95c28991d5b876c0ca0a87201439af1ee878ebd Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 3 Jul 2014 10:54:40 -0700 Subject: [PATCH 02/16] Basic contant parameters and field access (through variables of the given field name). --- .../expression/ExpressionScriptBindings.java | 32 ++++++++++++++ .../ExpressionScriptEngineService.java | 41 +++++++++++++++--- .../ExpressionScriptExecutionException.java | 3 ++ .../ExpressionScriptFunctionValues.java | 25 +++++++++++ .../ExpressionScriptValueSource.java | 43 +++++++++++++++++++ .../search/lookup/DocLookup.java | 2 +- .../script/ExpressionScriptTest.java | 4 +- 7 files changed, 141 insertions(+), 9 deletions(-) create mode 100644 src/main/java/org/elasticsearch/script/expression/ExpressionScriptBindings.java create mode 100644 src/main/java/org/elasticsearch/script/expression/ExpressionScriptFunctionValues.java create mode 100644 src/main/java/org/elasticsearch/script/expression/ExpressionScriptValueSource.java diff --git a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptBindings.java b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptBindings.java new file mode 100644 index 0000000000000..6f756fc723289 --- /dev/null +++ b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptBindings.java @@ -0,0 +1,32 @@ +package org.elasticsearch.script.expression; + +import org.apache.lucene.expressions.Bindings; +import org.apache.lucene.queries.function.ValueSource; +import org.apache.lucene.queries.function.valuesource.DoubleConstValueSource; +import org.elasticsearch.index.fielddata.IndexFieldData; + +import java.util.HashMap; +import java.util.Map; + + +class ExpressionScriptBindings extends Bindings { + + Map variables = new HashMap<>(); + + void addConstant(String variable, double value) { + variables.put(variable, new DoubleConstValueSource(value)); + } + + void addField(String variable, IndexFieldData fieldData) { + variables.put(variable, new ExpressionScriptValueSource(fieldData)); + } + + @Override + public ValueSource getValueSource(String variable) { + if (variable.equals("_score")) { + return getScoreValueSource(); + } else { + return variables.get(variable); + } + } +} diff --git a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java index 604308b065653..c188dd22cf8b3 100644 --- a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java +++ b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java @@ -2,12 +2,17 @@ import org.apache.lucene.expressions.Expression; import org.apache.lucene.expressions.SimpleBindings; +import org.apache.lucene.index.AtomicReaderContext; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.fielddata.AtomicNumericFieldData; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.index.mapper.core.NumberFieldMapper; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.ScriptEngineService; import org.elasticsearch.script.SearchScript; @@ -26,13 +31,11 @@ public class ExpressionScriptEngineService extends AbstractComponent implements private final AtomicLong counter = new AtomicLong(); private final ClassLoader classLoader; // TODO: should use this instead of the implicit this.getClass().getClassLoader()? - private final MapperService mapper; @Inject - public ExpressionScriptEngineService(Settings settings, MapperService m) { + public ExpressionScriptEngineService(Settings settings) { super(settings); classLoader = settings.getClassLoader(); - mapper = m; } @Override @@ -53,7 +56,7 @@ public boolean sandboxed() { @Override public Object compile(String script) { try { - // NOTE: validation is delayed to allow runtime vars + // NOTE: validation is delayed to allow runtime vars, and we don't have access to per index stuff here return JavascriptCompiler.compile(script); } catch (ParseException e) { throw new ExpressionScriptCompilationException("Failed to parse expression: " + script, e); @@ -63,9 +66,35 @@ public Object compile(String script) { @Override public SearchScript search(Object compiledScript, SearchLookup lookup, @Nullable Map vars) { Expression expr = (Expression)compiledScript; + MapperService mapper = lookup.doc().mapperService(); + ExpressionScriptBindings bindings = new ExpressionScriptBindings(); + + for (String variable : expr.variables) { + if (variable.equals("_score")) { + // noop: our bindings inherently know how to deal with score + } else if (vars != null && vars.containsKey(variable)) { + Object value = vars.get(variable); + if (value instanceof Double) { + bindings.addConstant(variable, ((Double)value).doubleValue()); + } else if (value instanceof Long) { + bindings.addConstant(variable, ((Long)value).doubleValue()); + } else if (value instanceof Integer) { + bindings.addConstant(variable, ((Integer)value).doubleValue()); + } else { + throw new ExpressionScriptExecutionException("Parameter [" + variable + "] must be a numeric type (double or long)"); + } + } else { + FieldMapper field = mapper.smartNameFieldMapper(variable); + if (field.isNumeric() == false) { + // TODO: more context (which expression?) + throw new ExpressionScriptExecutionException("Field [" + variable + "] used in expression must be numeric"); + } + IndexFieldData fieldData = lookup.doc().fieldDataService.getForField((NumberFieldMapper)field); + bindings.addField(variable, fieldData); + } + } - // TODO: fill in vars into bindings, do validation - return new ExpressionScript((Expression)compiledScript, new SimpleBindings()); + return new ExpressionScript((Expression)compiledScript, bindings); } @Override diff --git a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptExecutionException.java b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptExecutionException.java index 12c6867c4960a..5d2cd3da578c6 100644 --- a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptExecutionException.java +++ b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptExecutionException.java @@ -9,4 +9,7 @@ public class ExpressionScriptExecutionException extends ElasticsearchException { public ExpressionScriptExecutionException(String msg, Throwable cause) { super(msg, cause); } + public ExpressionScriptExecutionException(String msg) { + super(msg); + } } diff --git a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptFunctionValues.java b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptFunctionValues.java new file mode 100644 index 0000000000000..9dd7899259c41 --- /dev/null +++ b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptFunctionValues.java @@ -0,0 +1,25 @@ +package org.elasticsearch.script.expression; + +import org.apache.lucene.queries.function.FunctionValues; +import org.apache.lucene.queries.function.ValueSource; +import org.apache.lucene.queries.function.docvalues.DoubleDocValues; +import org.elasticsearch.index.fielddata.AtomicNumericFieldData; +import org.elasticsearch.index.fielddata.DoubleValues; + + +class ExpressionScriptFunctionValues extends DoubleDocValues { + DoubleValues dataAccessor; + + ExpressionScriptFunctionValues(ValueSource parent, AtomicNumericFieldData d) { + super(parent); + dataAccessor = d.getDoubleValues(); + } + + @Override + public double doubleVal(int i) { + dataAccessor.setDocument(i); + // TODO: how are default values handled (if doc doesn't have value for this field?) + // TODO: how to handle nth value for array access in the future? + return dataAccessor.nextValue(); + } +} diff --git a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptValueSource.java b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptValueSource.java new file mode 100644 index 0000000000000..495a9ccb96310 --- /dev/null +++ b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptValueSource.java @@ -0,0 +1,43 @@ +package org.elasticsearch.script.expression; + + +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.queries.function.FunctionValues; +import org.apache.lucene.queries.function.ValueSource; +import org.elasticsearch.index.fielddata.AtomicFieldData; +import org.elasticsearch.index.fielddata.AtomicNumericFieldData; +import org.elasticsearch.index.fielddata.IndexFieldData; + +import java.io.IOException; +import java.util.Map; + +class ExpressionScriptValueSource extends ValueSource { + + IndexFieldData fieldData; + + ExpressionScriptValueSource(IndexFieldData d) { + fieldData = d; + } + + @Override + public FunctionValues getValues(Map context, AtomicReaderContext leaf) throws IOException { + AtomicFieldData leafData = fieldData.load(leaf); + assert(leafData instanceof AtomicNumericFieldData); + return new ExpressionScriptFunctionValues(this, (AtomicNumericFieldData)leafData); + } + + @Override + public boolean equals(Object other) { + return fieldData.equals(other); + } + + @Override + public int hashCode() { + return fieldData.hashCode(); + } + + @Override + public String description() { + return "field(" + fieldData.getFieldNames().toString() + ")"; + } +} diff --git a/src/main/java/org/elasticsearch/search/lookup/DocLookup.java b/src/main/java/org/elasticsearch/search/lookup/DocLookup.java index 94fe9cbae4690..6970cdf744f3f 100644 --- a/src/main/java/org/elasticsearch/search/lookup/DocLookup.java +++ b/src/main/java/org/elasticsearch/search/lookup/DocLookup.java @@ -42,7 +42,7 @@ public class DocLookup implements Map { private final Map localCacheFieldData = Maps.newHashMapWithExpectedSize(4); private final MapperService mapperService; - private final IndexFieldDataService fieldDataService; + public final IndexFieldDataService fieldDataService; @Nullable private final String[] types; diff --git a/src/test/java/org/elasticsearch/script/ExpressionScriptTest.java b/src/test/java/org/elasticsearch/script/ExpressionScriptTest.java index f8dcc2cb2379c..31d8ed2e9e3db 100644 --- a/src/test/java/org/elasticsearch/script/ExpressionScriptTest.java +++ b/src/test/java/org/elasticsearch/script/ExpressionScriptTest.java @@ -9,7 +9,7 @@ public class ExpressionScriptTest extends ElasticsearchIntegrationTest { @Test public void testBasic() { client().prepareIndex("test", "doc", "1").setSource("foo", 5).setRefresh(true).get(); - String script = "1 + 2"; + String script = "foo + 2"; SearchResponse resp = client().prepareSearch("test") .setSource("{\"query\": {" + "\"function_score\": {" + @@ -22,7 +22,7 @@ public void testBasic() { "}" + "}}").get(); - assertEquals(3, resp.getHits().getAt(0).getScore(), 0.0001f); + assertEquals(7.0, resp.getHits().getAt(0).getScore(), 0.0001f); } From dfae59b226b06427b04b093e40206d578ac04b96 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 8 Jul 2014 08:26:20 -0700 Subject: [PATCH 03/16] Fixed _score to actually work --- .../script/expression/ExpressionScript.java | 73 ++++++++++++++----- .../expression/ExpressionScriptBindings.java | 5 +- .../ExpressionScriptEngineService.java | 7 +- .../script/ExpressionScriptTest.java | 18 ++--- 4 files changed, 74 insertions(+), 29 deletions(-) diff --git a/src/main/java/org/elasticsearch/script/expression/ExpressionScript.java b/src/main/java/org/elasticsearch/script/expression/ExpressionScript.java index 864b392411737..fa3f7b6d898ce 100644 --- a/src/main/java/org/elasticsearch/script/expression/ExpressionScript.java +++ b/src/main/java/org/elasticsearch/script/expression/ExpressionScript.java @@ -9,29 +9,71 @@ import org.elasticsearch.script.SearchScript; import java.io.IOException; -import java.util.HashMap; +import java.util.Collections; import java.util.Map; /** - * + * A bridge to evaluate an {@link Expression} against {@link Bindings} in the context + * of a {@link SearchScript}. */ class ExpressionScript implements SearchScript { + /** Fake scorer for a single document */ + class CannedScorer extends Scorer { + protected int docid; + protected float score; + + public CannedScorer() { + super(null); + } + + @Override + public int docID() { + return docid; + } + + @Override + public float score() throws IOException { + return score; + } + + @Override + public int freq() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int nextDoc() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public int advance(int target) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public long cost() { + return 1; + } + } + Expression expression; Bindings bindings; AtomicReaderContext leaf; - int docid; + CannedScorer scorer; ExpressionScript(Expression e, Bindings b) { expression = e; bindings = b; + scorer = new CannedScorer(); } double evaluate() { try { ValueSource vs = expression.getValueSource(bindings); - FunctionValues fv = vs.getValues(new HashMap(), leaf); - return fv.doubleVal(docid); + FunctionValues fv = vs.getValues(Collections.singletonMap("scorer", scorer), leaf); + return fv.doubleVal(scorer.docid); } catch (IOException e) { throw new ExpressionScriptExecutionException("Failed to run expression", e); } @@ -54,34 +96,29 @@ class ExpressionScript implements SearchScript { @Override public void setNextDocId(int d) { - docid = d; + scorer.docid = d; } + @Override + public void setNextScore(float score) { scorer.score = score; } + @Override public void setNextReader(AtomicReaderContext l) { leaf = l; } @Override - public void setNextSource(Map source) { - - } + public void setScorer(Scorer s) { /* noop: score isn't actually set for scoring... */ } @Override - public void setNextScore(float score) { - + public void setNextSource(Map source) { + // noop: expressions don't use source data } - @Override public void setNextVar(String name, Object value) { - + // nocommit: comment on why this isn't needed...wtf is it? } - - @Override - public void setScorer(Scorer scorer) { - - } } diff --git a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptBindings.java b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptBindings.java index 6f756fc723289..0d2e8ee4c9ae9 100644 --- a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptBindings.java +++ b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptBindings.java @@ -8,7 +8,9 @@ import java.util.HashMap; import java.util.Map; - +/** + * TODO: We could get rid of this entirely if SimpleBindings had add(String, ValueSource) instead of only add(SortField) + */ class ExpressionScriptBindings extends Bindings { Map variables = new HashMap<>(); @@ -23,6 +25,7 @@ void addField(String variable, IndexFieldData fieldData) { @Override public ValueSource getValueSource(String variable) { + // TODO: is _score a constant anywhere? if (variable.equals("_score")) { return getScoreValueSource(); } else { diff --git a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java index c188dd22cf8b3..151df47bd9168 100644 --- a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java +++ b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java @@ -25,7 +25,8 @@ import java.util.concurrent.atomic.AtomicLong; /** - * Provides the infrastructure for Lucene expressions as a scripting language for Elasticsearch. + * Provides the infrastructure for Lucene expressions as a scripting language for Elasticsearch. Only + * {@link SearchScript}s are supported. */ public class ExpressionScriptEngineService extends AbstractComponent implements ScriptEngineService { @@ -73,6 +74,9 @@ public SearchScript search(Object compiledScript, SearchLookup lookup, @Nullable if (variable.equals("_score")) { // noop: our bindings inherently know how to deal with score } else if (vars != null && vars.containsKey(variable)) { + // TODO: document and/or error if vars contains _score? + // NOTE: by checking for the variable in vars first, it allows masking document fields with a global constant, + // but if we were to reverse it, we could provide a way to supply dynamic defaults for documents missing the field? Object value = vars.get(variable); if (value instanceof Double) { bindings.addConstant(variable, ((Double)value).doubleValue()); @@ -84,6 +88,7 @@ public SearchScript search(Object compiledScript, SearchLookup lookup, @Nullable throw new ExpressionScriptExecutionException("Parameter [" + variable + "] must be a numeric type (double or long)"); } } else { + // TODO: extract field name/access pattern from variable FieldMapper field = mapper.smartNameFieldMapper(variable); if (field.isNumeric() == false) { // TODO: more context (which expression?) diff --git a/src/test/java/org/elasticsearch/script/ExpressionScriptTest.java b/src/test/java/org/elasticsearch/script/ExpressionScriptTest.java index 31d8ed2e9e3db..f09eb54009102 100644 --- a/src/test/java/org/elasticsearch/script/ExpressionScriptTest.java +++ b/src/test/java/org/elasticsearch/script/ExpressionScriptTest.java @@ -9,20 +9,20 @@ public class ExpressionScriptTest extends ElasticsearchIntegrationTest { @Test public void testBasic() { client().prepareIndex("test", "doc", "1").setSource("foo", 5).setRefresh(true).get(); - String script = "foo + 2"; + String script = "_score + foo + 2"; SearchResponse resp = client().prepareSearch("test") - .setSource("{\"query\": {" + - "\"function_score\": {" + - "\"query\":{\"match_all\":{}}," + - "\"boost_mode\": \"replace\"," + - "\"script_score\": {" + - "\"script\": \"" + script +"\"," + - "\"lang\": \"expression\"" + + .setSource("{query: {" + + "function_score: {" + + "query:{match_all:{}}," + + "boost_mode: \"replace\"," + + "script_score: {" + + "script: \"" + script +"\"," + + "lang: \"expression\"" + "}" + "}" + "}}").get(); - assertEquals(7.0, resp.getHits().getAt(0).getScore(), 0.0001f); + assertEquals(8.0, resp.getHits().getAt(0).getScore(), 0.0001f); } From 892d980957c11945f71c6a024c4d1fa3e0b49510 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Wed, 9 Jul 2014 15:17:17 -0700 Subject: [PATCH 04/16] Added copies changes from LUCENE-5806. Finished tests. --- .../lucene/expressions/XSimpleBindings.java | 123 + .../expressions/js/XJavascriptCompiler.java | 610 +++++ .../expressions/js/XJavascriptLexer.java | 2225 +++++++++++++++++ .../expressions/js/XJavascriptParser.java | 1971 +++++++++++++++ .../expressions/js/XVariableContext.java | 101 + .../script/expression/ExpressionScript.java | 23 +- .../expression/ExpressionScriptBindings.java | 35 - .../ExpressionScriptCompilationException.java | 19 + .../ExpressionScriptEngineService.java | 68 +- .../ExpressionScriptExecutionException.java | 19 + .../ExpressionScriptFunctionValues.java | 31 +- .../ExpressionScriptValueSource.java | 22 + .../script/ExpressionScriptTest.java | 29 - .../expression/ExpressionScriptTests.java | 186 ++ 14 files changed, 5370 insertions(+), 92 deletions(-) create mode 100644 src/main/java/org/apache/lucene/expressions/XSimpleBindings.java create mode 100644 src/main/java/org/apache/lucene/expressions/js/XJavascriptCompiler.java create mode 100644 src/main/java/org/apache/lucene/expressions/js/XJavascriptLexer.java create mode 100644 src/main/java/org/apache/lucene/expressions/js/XJavascriptParser.java create mode 100644 src/main/java/org/apache/lucene/expressions/js/XVariableContext.java delete mode 100644 src/main/java/org/elasticsearch/script/expression/ExpressionScriptBindings.java delete mode 100644 src/test/java/org/elasticsearch/script/ExpressionScriptTest.java create mode 100644 src/test/java/org/elasticsearch/script/expression/ExpressionScriptTests.java diff --git a/src/main/java/org/apache/lucene/expressions/XSimpleBindings.java b/src/main/java/org/apache/lucene/expressions/XSimpleBindings.java new file mode 100644 index 0000000000000..d72177e2bd1ea --- /dev/null +++ b/src/main/java/org/apache/lucene/expressions/XSimpleBindings.java @@ -0,0 +1,123 @@ +package org.apache.lucene.expressions; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.HashMap; +import java.util.Map; + +import org.apache.lucene.queries.function.ValueSource; +import org.apache.lucene.queries.function.valuesource.DoubleFieldSource; +import org.apache.lucene.queries.function.valuesource.FloatFieldSource; +import org.apache.lucene.queries.function.valuesource.IntFieldSource; +import org.apache.lucene.queries.function.valuesource.LongFieldSource; +import org.apache.lucene.search.SortField; + +/** + * Simple class that binds expression variable names to {@link SortField}s + * or other {@link Expression}s. + *

+ * Example usage: + * 

+ *   XSimpleBindings bindings = new XSimpleBindings();
+ *   // document's text relevance score
+ *   bindings.add(new SortField("_score", SortField.Type.SCORE));
+ *   // integer NumericDocValues field (or from FieldCache) 
+ *   bindings.add(new SortField("popularity", SortField.Type.INT));
+ *   // another expression
+ *   bindings.add("recency", myRecencyExpression);
+ *
+ *   // create a sort field in reverse order
+ *   Sort sort = new Sort(expr.getSortField(bindings, true));
+ *
+ * + * @lucene.experimental + */ +public final class XSimpleBindings extends Bindings { + final Map map = new HashMap<>(); + + /** Creates a new empty Bindings */ + public XSimpleBindings() {} + + /** + * Adds a SortField to the bindings. + *

+ * This can be used to reference a DocValuesField, a field from + * FieldCache, the document's score, etc. + */ + public void add(SortField sortField) { + map.put(sortField.getField(), sortField); + } + + /** + * Bind a {@link ValueSource} directly to the given name. + */ + public void add(String name, ValueSource source) { map.put(name, source); } + + /** + * Adds an Expression to the bindings. + *

+ * This can be used to reference expressions from other expressions. + */ + public void add(String name, Expression expression) { + map.put(name, expression); + } + + @Override + public ValueSource getValueSource(String name) { + Object o = map.get(name); + if (o == null) { + throw new IllegalArgumentException("Invalid reference '" + name + "'"); + } else if (o instanceof Expression) { + return ((Expression)o).getValueSource(this); + } else if (o instanceof ValueSource) { + return ((ValueSource)o); + } + SortField field = (SortField) o; + switch(field.getType()) { + case INT: + return new IntFieldSource(field.getField()); + case LONG: + return new LongFieldSource(field.getField()); + case FLOAT: + return new FloatFieldSource(field.getField()); + case DOUBLE: + return new DoubleFieldSource(field.getField()); + case SCORE: + return getScoreValueSource(); + default: + throw new UnsupportedOperationException(); + } + } + + /** + * Traverses the graph of bindings, checking there are no cycles or missing references + * @throws IllegalArgumentException if the bindings is inconsistent + */ + public void validate() { + for (Object o : map.values()) { + if (o instanceof Expression) { + Expression expr = (Expression) o; + try { + expr.getValueSource(this); + } catch (StackOverflowError e) { + throw new IllegalArgumentException("Recursion Error: Cycle detected originating in (" + expr.sourceText + ")"); + } + } + } + } +} diff --git a/src/main/java/org/apache/lucene/expressions/js/XJavascriptCompiler.java b/src/main/java/org/apache/lucene/expressions/js/XJavascriptCompiler.java new file mode 100644 index 0000000000000..60afa9efbbbce --- /dev/null +++ b/src/main/java/org/apache/lucene/expressions/js/XJavascriptCompiler.java @@ -0,0 +1,610 @@ +package org.apache.lucene.expressions.js; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.io.Reader; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.nio.charset.StandardCharsets; +import java.text.ParseException; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Properties; + +import org.antlr.runtime.ANTLRStringStream; +import org.antlr.runtime.CharStream; +import org.antlr.runtime.CommonTokenStream; +import org.antlr.runtime.RecognitionException; +import org.antlr.runtime.tree.Tree; +import org.apache.lucene.expressions.Expression; +import org.apache.lucene.queries.function.FunctionValues; +import org.apache.lucene.util.IOUtils; +import org.objectweb.asm.ClassWriter; +import org.objectweb.asm.Label; +import org.objectweb.asm.Opcodes; +import org.objectweb.asm.Type; +import org.objectweb.asm.commons.GeneratorAdapter; + +/** + * An expression compiler for javascript expressions. + *

+ * Example: + * 

+ *   Expression foo = XJavascriptCompiler.compile("((0.3*popularity)/10.0)+(0.7*score)");
+ *
+ *

+ * See the {@link org.apache.lucene.expressions.js package documentation} for + * the supported syntax and default functions. + *

+ * You can compile with an alternate set of functions via {@link #compile(String, Map, ClassLoader)}. + * For example: + * 

+ *   Map<String,Method> functions = new HashMap<>();
+ *   // add all the default functions
+ *   functions.putAll(XJavascriptCompiler.DEFAULT_FUNCTIONS);
+ *   // add cbrt()
+ *   functions.put("cbrt", Math.class.getMethod("cbrt", double.class));
+ *   // call compile with customized function map
+ *   Expression foo = XJavascriptCompiler.compile("cbrt(score)+ln(popularity)", 
+ *                                               functions, 
+ *                                               getClass().getClassLoader());
+ *
+ * + * @lucene.experimental + */ +public class XJavascriptCompiler { + + static final class Loader extends ClassLoader { + Loader(ClassLoader parent) { + super(parent); + } + + public Class define(String className, byte[] bytecode) { + return defineClass(className, bytecode, 0, bytecode.length).asSubclass(Expression.class); + } + } + + private static final int CLASSFILE_VERSION = Opcodes.V1_7; + + // We use the same class name for all generated classes as they all have their own class loader. + // The source code is displayed as "source file name" in stack trace. + private static final String COMPILED_EXPRESSION_CLASS = XJavascriptCompiler.class.getName() + "$CompiledExpression"; + private static final String COMPILED_EXPRESSION_INTERNAL = COMPILED_EXPRESSION_CLASS.replace('.', '/'); + + private static final Type EXPRESSION_TYPE = Type.getType(Expression.class); + private static final Type FUNCTION_VALUES_TYPE = Type.getType(FunctionValues.class); + + private static final org.objectweb.asm.commons.Method + EXPRESSION_CTOR = getMethod("void (String, String[])"), + EVALUATE_METHOD = getMethod("double evaluate(int, " + FunctionValues.class.getName() + "[])"), + DOUBLE_VAL_METHOD = getMethod("double doubleVal(int)"); + + // to work around import clash: + private static org.objectweb.asm.commons.Method getMethod(String method) { + return org.objectweb.asm.commons.Method.getMethod(method); + } + + // This maximum length is theoretically 65535 bytes, but as its CESU-8 encoded we dont know how large it is in bytes, so be safe + // rcmuir: "If your ranking function is that large you need to check yourself into a mental institution!" + private static final int MAX_SOURCE_LENGTH = 16384; + + private final String sourceText; + private final Map externalsMap = new LinkedHashMap<>(); + private final ClassWriter classWriter = new ClassWriter(ClassWriter.COMPUTE_FRAMES | ClassWriter.COMPUTE_MAXS); + private GeneratorAdapter gen; + + private final Map functions; + + /** + * Compiles the given expression. + * + * @param sourceText The expression to compile + * @return A new compiled expression + * @throws ParseException on failure to compile + */ + public static Expression compile(String sourceText) throws ParseException { + return new XJavascriptCompiler(sourceText).compileExpression(XJavascriptCompiler.class.getClassLoader()); + } + + /** + * Compiles the given expression with the supplied custom functions. + *

+ * Functions must be {@code public static}, return {@code double} and + * can take from zero to 256 {@code double} parameters. + * + * @param sourceText The expression to compile + * @param functions map of String names to functions + * @param parent a {@code ClassLoader} that should be used as the parent of the loaded class. + * It must contain all classes referred to by the given {@code functions}. + * @return A new compiled expression + * @throws ParseException on failure to compile + */ + public static Expression compile(String sourceText, Map functions, ClassLoader parent) throws ParseException { + if (parent == null) { + throw new NullPointerException("A parent ClassLoader must be given."); + } + for (Method m : functions.values()) { + checkFunction(m, parent); + } + return new XJavascriptCompiler(sourceText, functions).compileExpression(parent); + } + + /** + * This method is unused, it is just here to make sure that the function signatures don't change. + * If this method fails to compile, you also have to change the byte code generator to correctly + * use the FunctionValues class. + */ + @SuppressWarnings({"unused", "null"}) + private static void unusedTestCompile() { + FunctionValues f = null; + double ret = f.doubleVal(2); + } + + /** + * Constructs a compiler for expressions. + * @param sourceText The expression to compile + */ + private XJavascriptCompiler(String sourceText) { + this(sourceText, DEFAULT_FUNCTIONS); + } + + /** + * Constructs a compiler for expressions with specific set of functions + * @param sourceText The expression to compile + */ + private XJavascriptCompiler(String sourceText, Map functions) { + if (sourceText == null) { + throw new NullPointerException(); + } + this.sourceText = sourceText; + this.functions = functions; + } + + /** + * Compiles the given expression with the specified parent classloader + * + * @return A new compiled expression + * @throws ParseException on failure to compile + */ + private Expression compileExpression(ClassLoader parent) throws ParseException { + try { + Tree antlrTree = getAntlrComputedExpressionTree(); + + beginCompile(); + recursiveCompile(antlrTree, Type.DOUBLE_TYPE); + endCompile(); + + Class evaluatorClass = new Loader(parent) + .define(COMPILED_EXPRESSION_CLASS, classWriter.toByteArray()); + Constructor constructor = evaluatorClass.getConstructor(String.class, String[].class); + return constructor.newInstance(sourceText, externalsMap.keySet().toArray(new String[externalsMap.size()])); + } catch (InstantiationException | IllegalAccessException | NoSuchMethodException | InvocationTargetException exception) { + throw new IllegalStateException("An internal error occurred attempting to compile the expression (" + sourceText + ").", exception); + } + } + + private void beginCompile() { + classWriter.visit(CLASSFILE_VERSION, + Opcodes.ACC_PUBLIC | Opcodes.ACC_SUPER | Opcodes.ACC_FINAL | Opcodes.ACC_SYNTHETIC, + COMPILED_EXPRESSION_INTERNAL, + null, EXPRESSION_TYPE.getInternalName(), null); + String clippedSourceText = (sourceText.length() <= MAX_SOURCE_LENGTH) ? + sourceText : (sourceText.substring(0, MAX_SOURCE_LENGTH - 3) + "..."); + classWriter.visitSource(clippedSourceText, null); + + GeneratorAdapter constructor = new GeneratorAdapter(Opcodes.ACC_PUBLIC | Opcodes.ACC_SYNTHETIC, + EXPRESSION_CTOR, null, null, classWriter); + constructor.loadThis(); + constructor.loadArgs(); + constructor.invokeConstructor(EXPRESSION_TYPE, EXPRESSION_CTOR); + constructor.returnValue(); + constructor.endMethod(); + + gen = new GeneratorAdapter(Opcodes.ACC_PUBLIC | Opcodes.ACC_SYNTHETIC, + EVALUATE_METHOD, null, null, classWriter); + } + + private void recursiveCompile(Tree current, Type expected) { + int type = current.getType(); + String text = current.getText(); + + switch (type) { + case XJavascriptParser.AT_CALL: + Tree identifier = current.getChild(0); + String call = identifier.getText(); + int arguments = current.getChildCount() - 1; + + Method method = functions.get(call); + if (method == null) { + throw new IllegalArgumentException("Unrecognized method call (" + call + ")."); + } + + int arity = method.getParameterTypes().length; + if (arguments != arity) { + throw new IllegalArgumentException("Expected (" + arity + ") arguments for method call (" + + call + "), but found (" + arguments + ")."); + } + + for (int argument = 1; argument <= arguments; ++argument) { + recursiveCompile(current.getChild(argument), Type.DOUBLE_TYPE); + } + + gen.invokeStatic(Type.getType(method.getDeclaringClass()), + org.objectweb.asm.commons.Method.getMethod(method)); + + gen.cast(Type.DOUBLE_TYPE, expected); + break; + case XJavascriptParser.VARIABLE: + int index; + + // normalize quotes + text = normalizeQuotes(text); + + + if (externalsMap.containsKey(text)) { + index = externalsMap.get(text); + } else { + index = externalsMap.size(); + externalsMap.put(text, index); + } + + gen.loadArg(1); + gen.push(index); + gen.arrayLoad(FUNCTION_VALUES_TYPE); + gen.loadArg(0); + gen.invokeVirtual(FUNCTION_VALUES_TYPE, DOUBLE_VAL_METHOD); + gen.cast(Type.DOUBLE_TYPE, expected); + break; + case XJavascriptParser.HEX: + pushLong(expected, Long.parseLong(text.substring(2), 16)); + break; + case XJavascriptParser.OCTAL: + pushLong(expected, Long.parseLong(text.substring(1), 8)); + break; + case XJavascriptParser.DECIMAL: + gen.push(Double.parseDouble(text)); + gen.cast(Type.DOUBLE_TYPE, expected); + break; + case XJavascriptParser.AT_NEGATE: + recursiveCompile(current.getChild(0), Type.DOUBLE_TYPE); + gen.visitInsn(Opcodes.DNEG); + gen.cast(Type.DOUBLE_TYPE, expected); + break; + case XJavascriptParser.AT_ADD: + pushArith(Opcodes.DADD, current, expected); + break; + case XJavascriptParser.AT_SUBTRACT: + pushArith(Opcodes.DSUB, current, expected); + break; + case XJavascriptParser.AT_MULTIPLY: + pushArith(Opcodes.DMUL, current, expected); + break; + case XJavascriptParser.AT_DIVIDE: + pushArith(Opcodes.DDIV, current, expected); + break; + case XJavascriptParser.AT_MODULO: + pushArith(Opcodes.DREM, current, expected); + break; + case XJavascriptParser.AT_BIT_SHL: + pushShift(Opcodes.LSHL, current, expected); + break; + case XJavascriptParser.AT_BIT_SHR: + pushShift(Opcodes.LSHR, current, expected); + break; + case XJavascriptParser.AT_BIT_SHU: + pushShift(Opcodes.LUSHR, current, expected); + break; + case XJavascriptParser.AT_BIT_AND: + pushBitwise(Opcodes.LAND, current, expected); + break; + case XJavascriptParser.AT_BIT_OR: + pushBitwise(Opcodes.LOR, current, expected); + break; + case XJavascriptParser.AT_BIT_XOR: + pushBitwise(Opcodes.LXOR, current, expected); + break; + case XJavascriptParser.AT_BIT_NOT: + recursiveCompile(current.getChild(0), Type.LONG_TYPE); + gen.push(-1L); + gen.visitInsn(Opcodes.LXOR); + gen.cast(Type.LONG_TYPE, expected); + break; + case XJavascriptParser.AT_COMP_EQ: + pushCond(GeneratorAdapter.EQ, current, expected); + break; + case XJavascriptParser.AT_COMP_NEQ: + pushCond(GeneratorAdapter.NE, current, expected); + break; + case XJavascriptParser.AT_COMP_LT: + pushCond(GeneratorAdapter.LT, current, expected); + break; + case XJavascriptParser.AT_COMP_GT: + pushCond(GeneratorAdapter.GT, current, expected); + break; + case XJavascriptParser.AT_COMP_LTE: + pushCond(GeneratorAdapter.LE, current, expected); + break; + case XJavascriptParser.AT_COMP_GTE: + pushCond(GeneratorAdapter.GE, current, expected); + break; + case XJavascriptParser.AT_BOOL_NOT: + Label labelNotTrue = new Label(); + Label labelNotReturn = new Label(); + + recursiveCompile(current.getChild(0), Type.INT_TYPE); + gen.visitJumpInsn(Opcodes.IFEQ, labelNotTrue); + pushBoolean(expected, false); + gen.goTo(labelNotReturn); + gen.visitLabel(labelNotTrue); + pushBoolean(expected, true); + gen.visitLabel(labelNotReturn); + break; + case XJavascriptParser.AT_BOOL_AND: + Label andFalse = new Label(); + Label andEnd = new Label(); + + recursiveCompile(current.getChild(0), Type.INT_TYPE); + gen.visitJumpInsn(Opcodes.IFEQ, andFalse); + recursiveCompile(current.getChild(1), Type.INT_TYPE); + gen.visitJumpInsn(Opcodes.IFEQ, andFalse); + pushBoolean(expected, true); + gen.goTo(andEnd); + gen.visitLabel(andFalse); + pushBoolean(expected, false); + gen.visitLabel(andEnd); + break; + case XJavascriptParser.AT_BOOL_OR: + Label orTrue = new Label(); + Label orEnd = new Label(); + + recursiveCompile(current.getChild(0), Type.INT_TYPE); + gen.visitJumpInsn(Opcodes.IFNE, orTrue); + recursiveCompile(current.getChild(1), Type.INT_TYPE); + gen.visitJumpInsn(Opcodes.IFNE, orTrue); + pushBoolean(expected, false); + gen.goTo(orEnd); + gen.visitLabel(orTrue); + pushBoolean(expected, true); + gen.visitLabel(orEnd); + break; + case XJavascriptParser.AT_COND_QUE: + Label condFalse = new Label(); + Label condEnd = new Label(); + + recursiveCompile(current.getChild(0), Type.INT_TYPE); + gen.visitJumpInsn(Opcodes.IFEQ, condFalse); + recursiveCompile(current.getChild(1), expected); + gen.goTo(condEnd); + gen.visitLabel(condFalse); + recursiveCompile(current.getChild(2), expected); + gen.visitLabel(condEnd); + break; + default: + throw new IllegalStateException("Unknown operation specified: (" + current.getText() + ")."); + } + } + + private void pushArith(int operator, Tree current, Type expected) { + pushBinaryOp(operator, current, expected, Type.DOUBLE_TYPE, Type.DOUBLE_TYPE, Type.DOUBLE_TYPE); + } + + private void pushShift(int operator, Tree current, Type expected) { + pushBinaryOp(operator, current, expected, Type.LONG_TYPE, Type.INT_TYPE, Type.LONG_TYPE); + } + + private void pushBitwise(int operator, Tree current, Type expected) { + pushBinaryOp(operator, current, expected, Type.LONG_TYPE, Type.LONG_TYPE, Type.LONG_TYPE); + } + + private void pushBinaryOp(int operator, Tree current, Type expected, Type arg1, Type arg2, Type returnType) { + recursiveCompile(current.getChild(0), arg1); + recursiveCompile(current.getChild(1), arg2); + gen.visitInsn(operator); + gen.cast(returnType, expected); + } + + private void pushCond(int operator, Tree current, Type expected) { + Label labelTrue = new Label(); + Label labelReturn = new Label(); + + recursiveCompile(current.getChild(0), Type.DOUBLE_TYPE); + recursiveCompile(current.getChild(1), Type.DOUBLE_TYPE); + + gen.ifCmp(Type.DOUBLE_TYPE, operator, labelTrue); + pushBoolean(expected, false); + gen.goTo(labelReturn); + gen.visitLabel(labelTrue); + pushBoolean(expected, true); + gen.visitLabel(labelReturn); + } + + private void pushBoolean(Type expected, boolean truth) { + switch (expected.getSort()) { + case Type.INT: + gen.push(truth); + break; + case Type.LONG: + gen.push(truth ? 1L : 0L); + break; + case Type.DOUBLE: + gen.push(truth ? 1. : 0.); + break; + default: + throw new IllegalStateException("Invalid expected type: " + expected); + } + } + + private void pushLong(Type expected, long i) { + switch (expected.getSort()) { + case Type.INT: + gen.push((int) i); + break; + case Type.LONG: + gen.push(i); + break; + case Type.DOUBLE: + gen.push((double) i); + break; + default: + throw new IllegalStateException("Invalid expected type: " + expected); + } + } + + private void endCompile() { + gen.returnValue(); + gen.endMethod(); + + classWriter.visitEnd(); + } + + private Tree getAntlrComputedExpressionTree() throws ParseException { + CharStream input = new ANTLRStringStream(sourceText); + XJavascriptLexer lexer = new XJavascriptLexer(input); + CommonTokenStream tokens = new CommonTokenStream(lexer); + XJavascriptParser parser = new XJavascriptParser(tokens); + + try { + return parser.expression().tree; + + } catch (RecognitionException exception) { + throw new IllegalArgumentException(exception); + } catch (RuntimeException exception) { + if (exception.getCause() instanceof ParseException) { + throw (ParseException)exception.getCause(); + } + throw exception; + } + } + + private static String normalizeQuotes(String text) { + StringBuilder out = new StringBuilder(text.length()); + boolean inDoubleQuotes = false; + for (int i = 0; i < text.length(); ++i) { + char c = text.charAt(i); + if (c == '\\') { + c = text.charAt(++i); + if (c == '\\') { + out.append('\\'); // re-escape the backslash + } + // no escape for double quote + } else if (c == '\'') { + if (inDoubleQuotes) { + // escape in output + out.append('\\'); + } else { + int j = findSingleQuoteStringEnd(text, i); + out.append(text, i, j); // copy up to end quote (leave end for append below) + i = j; + } + } else if (c == '"') { + c = '\''; // change beginning/ending doubles to singles + inDoubleQuotes = !inDoubleQuotes; + } + out.append(c); + } + return out.toString(); + } + + private static int findSingleQuoteStringEnd(String text, int start) { + ++start; // skip beginning + while (text.charAt(start) != '\'') { + if (text.charAt(start) == '\\') { + ++start; // blindly consume escape value + } + ++start; + } + return start; + } + + /** + * The default set of functions available to expressions. + *

+ * See the {@link org.apache.lucene.expressions.js package documentation} + * for a list. + */ + public static final Map DEFAULT_FUNCTIONS; + static { + Map map = new HashMap<>(); + try { + final Properties props = new Properties(); + try (Reader in = IOUtils.getDecodingReader(JavascriptCompiler.class, + JavascriptCompiler.class.getSimpleName() + ".properties", StandardCharsets.UTF_8)) { + props.load(in); + } + for (final String call : props.stringPropertyNames()) { + final String[] vals = props.getProperty(call).split(","); + if (vals.length != 3) { + throw new Error("Syntax error while reading Javascript functions from resource"); + } + final Class clazz = Class.forName(vals[0].trim()); + final String methodName = vals[1].trim(); + final int arity = Integer.parseInt(vals[2].trim()); + @SuppressWarnings({"rawtypes", "unchecked"}) Class[] args = new Class[arity]; + Arrays.fill(args, double.class); + Method method = clazz.getMethod(methodName, args); + checkFunction(method, JavascriptCompiler.class.getClassLoader()); + map.put(call, method); + } + } catch (NoSuchMethodException | ClassNotFoundException | IOException e) { + throw new Error("Cannot resolve function", e); + } + DEFAULT_FUNCTIONS = Collections.unmodifiableMap(map); + } + + private static void checkFunction(Method method, ClassLoader parent) { + // We can only call the function if the given parent class loader of our compiled class has access to the method: + final ClassLoader functionClassloader = method.getDeclaringClass().getClassLoader(); + if (functionClassloader != null) { // it is a system class iff null! + boolean found = false; + while (parent != null) { + if (parent == functionClassloader) { + found = true; + break; + } + parent = parent.getParent(); + } + if (!found) { + throw new IllegalArgumentException(method + " is not declared by a class which is accessible by the given parent ClassLoader."); + } + } + // do some checks if the signature is "compatible": + if (!Modifier.isStatic(method.getModifiers())) { + throw new IllegalArgumentException(method + " is not static."); + } + if (!Modifier.isPublic(method.getModifiers())) { + throw new IllegalArgumentException(method + " is not public."); + } + if (!Modifier.isPublic(method.getDeclaringClass().getModifiers())) { + throw new IllegalArgumentException(method.getDeclaringClass().getName() + " is not public."); + } + for (Class clazz : method.getParameterTypes()) { + if (!clazz.equals(double.class)) { + throw new IllegalArgumentException(method + " must take only double parameters"); + } + } + if (method.getReturnType() != double.class) { + throw new IllegalArgumentException(method + " does not return a double."); + } + } +} + diff --git a/src/main/java/org/apache/lucene/expressions/js/XJavascriptLexer.java b/src/main/java/org/apache/lucene/expressions/js/XJavascriptLexer.java new file mode 100644 index 0000000000000..f63a98dd4c522 --- /dev/null +++ b/src/main/java/org/apache/lucene/expressions/js/XJavascriptLexer.java @@ -0,0 +1,2225 @@ +// ANTLR GENERATED CODE: DO NOT EDIT + +package org.apache.lucene.expressions.js; + +import java.text.ParseException; + + +import org.antlr.runtime.*; +import java.util.Stack; +import java.util.List; +import java.util.ArrayList; + +@SuppressWarnings("all") +class XJavascriptLexer extends Lexer { + public static final int EOF=-1; + public static final int ARRAY=4; + public static final int AT_ADD=5; + public static final int AT_BIT_AND=6; + public static final int AT_BIT_NOT=7; + public static final int AT_BIT_OR=8; + public static final int AT_BIT_SHL=9; + public static final int AT_BIT_SHR=10; + public static final int AT_BIT_SHU=11; + public static final int AT_BIT_XOR=12; + public static final int AT_BOOL_AND=13; + public static final int AT_BOOL_NOT=14; + public static final int AT_BOOL_OR=15; + public static final int AT_CALL=16; + public static final int AT_COLON=17; + public static final int AT_COMMA=18; + public static final int AT_COMP_EQ=19; + public static final int AT_COMP_GT=20; + public static final int AT_COMP_GTE=21; + public static final int AT_COMP_LT=22; + public static final int AT_COMP_LTE=23; + public static final int AT_COMP_NEQ=24; + public static final int AT_COND_QUE=25; + public static final int AT_DIVIDE=26; + public static final int AT_DOT=27; + public static final int AT_LPAREN=28; + public static final int AT_MODULO=29; + public static final int AT_MULTIPLY=30; + public static final int AT_NEGATE=31; + public static final int AT_RPAREN=32; + public static final int AT_SUBTRACT=33; + public static final int DECIMAL=34; + public static final int DECIMALDIGIT=35; + public static final int DECIMALINTEGER=36; + public static final int DOUBLE_STRING_CHAR=37; + public static final int EXPONENT=38; + public static final int HEX=39; + public static final int HEXDIGIT=40; + public static final int ID=41; + public static final int OBJECT=42; + public static final int OCTAL=43; + public static final int OCTALDIGIT=44; + public static final int SINGLE_STRING_CHAR=45; + public static final int STRING=46; + public static final int VARIABLE=47; + public static final int WS=48; + + + @Override + public void displayRecognitionError(String[] tokenNames, RecognitionException re) { + String message = " unexpected character '" + (char)re.c + + "' at position (" + re.charPositionInLine + ")."; + ParseException parseException = new ParseException(message, re.charPositionInLine); + parseException.initCause(re); + throw new RuntimeException(parseException); + } + + + + // delegates + // delegators + public Lexer[] getDelegates() { + return new Lexer[] {}; + } + + public XJavascriptLexer() {} + public XJavascriptLexer(CharStream input) { + this(input, new RecognizerSharedState()); + } + public XJavascriptLexer(CharStream input, RecognizerSharedState state) { + super(input,state); + } + @Override public String getGrammarFileName() { return "src/java/org/apache/lucene/expressions/js/Javascript.g"; } + + // $ANTLR start "AT_ADD" + public final void mAT_ADD() throws RecognitionException { + try { + int _type = AT_ADD; + int _channel = DEFAULT_TOKEN_CHANNEL; + // src/java/org/apache/lucene/expressions/js/Javascript.g:25:8: ( '+' ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:25:10: '+' + { + match('+'); + } + + state.type = _type; + state.channel = _channel; + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "AT_ADD" + + // $ANTLR start "AT_BIT_AND" + public final void mAT_BIT_AND() throws RecognitionException { + try { + int _type = AT_BIT_AND; + int _channel = DEFAULT_TOKEN_CHANNEL; + // src/java/org/apache/lucene/expressions/js/Javascript.g:26:12: ( '&' ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:26:14: '&' + { + match('&'); + } + + state.type = _type; + state.channel = _channel; + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "AT_BIT_AND" + + // $ANTLR start "AT_BIT_NOT" + public final void mAT_BIT_NOT() throws RecognitionException { + try { + int _type = AT_BIT_NOT; + int _channel = DEFAULT_TOKEN_CHANNEL; + // src/java/org/apache/lucene/expressions/js/Javascript.g:27:12: ( '~' ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:27:14: '~' + { + match('~'); + } + + state.type = _type; + state.channel = _channel; + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "AT_BIT_NOT" + + // $ANTLR start "AT_BIT_OR" + public final void mAT_BIT_OR() throws RecognitionException { + try { + int _type = AT_BIT_OR; + int _channel = DEFAULT_TOKEN_CHANNEL; + // src/java/org/apache/lucene/expressions/js/Javascript.g:28:11: ( '|' ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:28:13: '|' + { + match('|'); + } + + state.type = _type; + state.channel = _channel; + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "AT_BIT_OR" + + // $ANTLR start "AT_BIT_SHL" + public final void mAT_BIT_SHL() throws RecognitionException { + try { + int _type = AT_BIT_SHL; + int _channel = DEFAULT_TOKEN_CHANNEL; + // src/java/org/apache/lucene/expressions/js/Javascript.g:29:12: ( '<<' ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:29:14: '<<' + { + match("<<"); + + } + + state.type = _type; + state.channel = _channel; + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "AT_BIT_SHL" + + // $ANTLR start "AT_BIT_SHR" + public final void mAT_BIT_SHR() throws RecognitionException { + try { + int _type = AT_BIT_SHR; + int _channel = DEFAULT_TOKEN_CHANNEL; + // src/java/org/apache/lucene/expressions/js/Javascript.g:30:12: ( '>>' ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:30:14: '>>' + { + match(">>"); + + } + + state.type = _type; + state.channel = _channel; + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "AT_BIT_SHR" + + // $ANTLR start "AT_BIT_SHU" + public final void mAT_BIT_SHU() throws RecognitionException { + try { + int _type = AT_BIT_SHU; + int _channel = DEFAULT_TOKEN_CHANNEL; + // src/java/org/apache/lucene/expressions/js/Javascript.g:31:12: ( '>>>' ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:31:14: '>>>' + { + match(">>>"); + + } + + state.type = _type; + state.channel = _channel; + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "AT_BIT_SHU" + + // $ANTLR start "AT_BIT_XOR" + public final void mAT_BIT_XOR() throws RecognitionException { + try { + int _type = AT_BIT_XOR; + int _channel = DEFAULT_TOKEN_CHANNEL; + // src/java/org/apache/lucene/expressions/js/Javascript.g:32:12: ( '^' ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:32:14: '^' + { + match('^'); + } + + state.type = _type; + state.channel = _channel; + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "AT_BIT_XOR" + + // $ANTLR start "AT_BOOL_AND" + public final void mAT_BOOL_AND() throws RecognitionException { + try { + int _type = AT_BOOL_AND; + int _channel = DEFAULT_TOKEN_CHANNEL; + // src/java/org/apache/lucene/expressions/js/Javascript.g:33:13: ( '&&' ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:33:15: '&&' + { + match("&&"); + + } + + state.type = _type; + state.channel = _channel; + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "AT_BOOL_AND" + + // $ANTLR start "AT_BOOL_NOT" + public final void mAT_BOOL_NOT() throws RecognitionException { + try { + int _type = AT_BOOL_NOT; + int _channel = DEFAULT_TOKEN_CHANNEL; + // src/java/org/apache/lucene/expressions/js/Javascript.g:34:13: ( '!' ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:34:15: '!' + { + match('!'); + } + + state.type = _type; + state.channel = _channel; + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "AT_BOOL_NOT" + + // $ANTLR start "AT_BOOL_OR" + public final void mAT_BOOL_OR() throws RecognitionException { + try { + int _type = AT_BOOL_OR; + int _channel = DEFAULT_TOKEN_CHANNEL; + // src/java/org/apache/lucene/expressions/js/Javascript.g:35:12: ( '||' ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:35:14: '||' + { + match("||"); + + } + + state.type = _type; + state.channel = _channel; + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "AT_BOOL_OR" + + // $ANTLR start "AT_COLON" + public final void mAT_COLON() throws RecognitionException { + try { + int _type = AT_COLON; + int _channel = DEFAULT_TOKEN_CHANNEL; + // src/java/org/apache/lucene/expressions/js/Javascript.g:36:10: ( ':' ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:36:12: ':' + { + match(':'); + } + + state.type = _type; + state.channel = _channel; + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "AT_COLON" + + // $ANTLR start "AT_COMMA" + public final void mAT_COMMA() throws RecognitionException { + try { + int _type = AT_COMMA; + int _channel = DEFAULT_TOKEN_CHANNEL; + // src/java/org/apache/lucene/expressions/js/Javascript.g:37:10: ( ',' ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:37:12: ',' + { + match(','); + } + + state.type = _type; + state.channel = _channel; + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "AT_COMMA" + + // $ANTLR start "AT_COMP_EQ" + public final void mAT_COMP_EQ() throws RecognitionException { + try { + int _type = AT_COMP_EQ; + int _channel = DEFAULT_TOKEN_CHANNEL; + // src/java/org/apache/lucene/expressions/js/Javascript.g:38:12: ( '==' ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:38:14: '==' + { + match("=="); + + } + + state.type = _type; + state.channel = _channel; + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "AT_COMP_EQ" + + // $ANTLR start "AT_COMP_GT" + public final void mAT_COMP_GT() throws RecognitionException { + try { + int _type = AT_COMP_GT; + int _channel = DEFAULT_TOKEN_CHANNEL; + // src/java/org/apache/lucene/expressions/js/Javascript.g:39:12: ( '>' ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:39:14: '>' + { + match('>'); + } + + state.type = _type; + state.channel = _channel; + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "AT_COMP_GT" + + // $ANTLR start "AT_COMP_GTE" + public final void mAT_COMP_GTE() throws RecognitionException { + try { + int _type = AT_COMP_GTE; + int _channel = DEFAULT_TOKEN_CHANNEL; + // src/java/org/apache/lucene/expressions/js/Javascript.g:40:13: ( '>=' ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:40:15: '>=' + { + match(">="); + + } + + state.type = _type; + state.channel = _channel; + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "AT_COMP_GTE" + + // $ANTLR start "AT_COMP_LT" + public final void mAT_COMP_LT() throws RecognitionException { + try { + int _type = AT_COMP_LT; + int _channel = DEFAULT_TOKEN_CHANNEL; + // src/java/org/apache/lucene/expressions/js/Javascript.g:41:12: ( '<' ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:41:14: '<' + { + match('<'); + } + + state.type = _type; + state.channel = _channel; + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "AT_COMP_LT" + + // $ANTLR start "AT_COMP_LTE" + public final void mAT_COMP_LTE() throws RecognitionException { + try { + int _type = AT_COMP_LTE; + int _channel = DEFAULT_TOKEN_CHANNEL; + // src/java/org/apache/lucene/expressions/js/Javascript.g:42:13: ( '<=' ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:42:15: '<=' + { + match("<="); + + } + + state.type = _type; + state.channel = _channel; + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "AT_COMP_LTE" + + // $ANTLR start "AT_COMP_NEQ" + public final void mAT_COMP_NEQ() throws RecognitionException { + try { + int _type = AT_COMP_NEQ; + int _channel = DEFAULT_TOKEN_CHANNEL; + // src/java/org/apache/lucene/expressions/js/Javascript.g:43:13: ( '!=' ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:43:15: '!=' + { + match("!="); + + } + + state.type = _type; + state.channel = _channel; + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "AT_COMP_NEQ" + + // $ANTLR start "AT_COND_QUE" + public final void mAT_COND_QUE() throws RecognitionException { + try { + int _type = AT_COND_QUE; + int _channel = DEFAULT_TOKEN_CHANNEL; + // src/java/org/apache/lucene/expressions/js/Javascript.g:44:13: ( '?' ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:44:15: '?' + { + match('?'); + } + + state.type = _type; + state.channel = _channel; + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "AT_COND_QUE" + + // $ANTLR start "AT_DIVIDE" + public final void mAT_DIVIDE() throws RecognitionException { + try { + int _type = AT_DIVIDE; + int _channel = DEFAULT_TOKEN_CHANNEL; + // src/java/org/apache/lucene/expressions/js/Javascript.g:45:11: ( '/' ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:45:13: '/' + { + match('/'); + } + + state.type = _type; + state.channel = _channel; + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "AT_DIVIDE" + + // $ANTLR start "AT_DOT" + public final void mAT_DOT() throws RecognitionException { + try { + int _type = AT_DOT; + int _channel = DEFAULT_TOKEN_CHANNEL; + // src/java/org/apache/lucene/expressions/js/Javascript.g:46:8: ( '.' ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:46:10: '.' + { + match('.'); + } + + state.type = _type; + state.channel = _channel; + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "AT_DOT" + + // $ANTLR start "AT_LPAREN" + public final void mAT_LPAREN() throws RecognitionException { + try { + int _type = AT_LPAREN; + int _channel = DEFAULT_TOKEN_CHANNEL; + // src/java/org/apache/lucene/expressions/js/Javascript.g:47:11: ( '(' ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:47:13: '(' + { + match('('); + } + + state.type = _type; + state.channel = _channel; + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "AT_LPAREN" + + // $ANTLR start "AT_MODULO" + public final void mAT_MODULO() throws RecognitionException { + try { + int _type = AT_MODULO; + int _channel = DEFAULT_TOKEN_CHANNEL; + // src/java/org/apache/lucene/expressions/js/Javascript.g:48:11: ( '%' ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:48:13: '%' + { + match('%'); + } + + state.type = _type; + state.channel = _channel; + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "AT_MODULO" + + // $ANTLR start "AT_MULTIPLY" + public final void mAT_MULTIPLY() throws RecognitionException { + try { + int _type = AT_MULTIPLY; + int _channel = DEFAULT_TOKEN_CHANNEL; + // src/java/org/apache/lucene/expressions/js/Javascript.g:49:13: ( '*' ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:49:15: '*' + { + match('*'); + } + + state.type = _type; + state.channel = _channel; + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "AT_MULTIPLY" + + // $ANTLR start "AT_RPAREN" + public final void mAT_RPAREN() throws RecognitionException { + try { + int _type = AT_RPAREN; + int _channel = DEFAULT_TOKEN_CHANNEL; + // src/java/org/apache/lucene/expressions/js/Javascript.g:50:11: ( ')' ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:50:13: ')' + { + match(')'); + } + + state.type = _type; + state.channel = _channel; + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "AT_RPAREN" + + // $ANTLR start "AT_SUBTRACT" + public final void mAT_SUBTRACT() throws RecognitionException { + try { + int _type = AT_SUBTRACT; + int _channel = DEFAULT_TOKEN_CHANNEL; + // src/java/org/apache/lucene/expressions/js/Javascript.g:51:13: ( '-' ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:51:15: '-' + { + match('-'); + } + + state.type = _type; + state.channel = _channel; + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "AT_SUBTRACT" + + // $ANTLR start "VARIABLE" + public final void mVARIABLE() throws RecognitionException { + try { + int _type = VARIABLE; + int _channel = DEFAULT_TOKEN_CHANNEL; + // src/java/org/apache/lucene/expressions/js/Javascript.g:334:5: ( OBJECT ( AT_DOT OBJECT )* ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:334:7: OBJECT ( AT_DOT OBJECT )* + { + mOBJECT(); + + // src/java/org/apache/lucene/expressions/js/Javascript.g:334:14: ( AT_DOT OBJECT )* + loop1: + while (true) { + int alt1=2; + int LA1_0 = input.LA(1); + if ( (LA1_0=='.') ) { + alt1=1; + } + + switch (alt1) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:334:15: AT_DOT OBJECT + { + mAT_DOT(); + + mOBJECT(); + + } + break; + + default : + break loop1; + } + } + + } + + state.type = _type; + state.channel = _channel; + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "VARIABLE" + + // $ANTLR start "OBJECT" + public final void mOBJECT() throws RecognitionException { + try { + // src/java/org/apache/lucene/expressions/js/Javascript.g:340:5: ( ID ( ARRAY )* ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:340:7: ID ( ARRAY )* + { + mID(); + + // src/java/org/apache/lucene/expressions/js/Javascript.g:340:10: ( ARRAY )* + loop2: + while (true) { + int alt2=2; + int LA2_0 = input.LA(1); + if ( (LA2_0=='[') ) { + alt2=1; + } + + switch (alt2) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:340:10: ARRAY + { + mARRAY(); + + } + break; + + default : + break loop2; + } + } + + } + + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "OBJECT" + + // $ANTLR start "ARRAY" + public final void mARRAY() throws RecognitionException { + try { + // src/java/org/apache/lucene/expressions/js/Javascript.g:345:5: ( '[' STRING ']' | '[' DECIMALINTEGER ']' ) + int alt3=2; + int LA3_0 = input.LA(1); + if ( (LA3_0=='[') ) { + int LA3_1 = input.LA(2); + if ( (LA3_1=='\"'||LA3_1=='\'') ) { + alt3=1; + } + else if ( ((LA3_1 >= '0' && LA3_1 <= '9')) ) { + alt3=2; + } + + else { + int nvaeMark = input.mark(); + try { + input.consume(); + NoViableAltException nvae = + new NoViableAltException("", 3, 1, input); + throw nvae; + } finally { + input.rewind(nvaeMark); + } + } + + } + + else { + NoViableAltException nvae = + new NoViableAltException("", 3, 0, input); + throw nvae; + } + + switch (alt3) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:345:7: '[' STRING ']' + { + match('['); + mSTRING(); + + match(']'); + } + break; + case 2 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:346:7: '[' DECIMALINTEGER ']' + { + match('['); + mDECIMALINTEGER(); + + match(']'); + } + break; + + } + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "ARRAY" + + // $ANTLR start "ID" + public final void mID() throws RecognitionException { + try { + // src/java/org/apache/lucene/expressions/js/Javascript.g:351:5: ( ( 'a' .. 'z' | 'A' .. 'Z' | '_' | '$' ) ( 'a' .. 'z' | 'A' .. 'Z' | '0' .. '9' | '_' | '$' )* ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:351:7: ( 'a' .. 'z' | 'A' .. 'Z' | '_' | '$' ) ( 'a' .. 'z' | 'A' .. 'Z' | '0' .. '9' | '_' | '$' )* + { + if ( input.LA(1)=='$'||(input.LA(1) >= 'A' && input.LA(1) <= 'Z')||input.LA(1)=='_'||(input.LA(1) >= 'a' && input.LA(1) <= 'z') ) { + input.consume(); + } + else { + MismatchedSetException mse = new MismatchedSetException(null,input); + recover(mse); + throw mse; + } + // src/java/org/apache/lucene/expressions/js/Javascript.g:351:35: ( 'a' .. 'z' | 'A' .. 'Z' | '0' .. '9' | '_' | '$' )* + loop4: + while (true) { + int alt4=2; + int LA4_0 = input.LA(1); + if ( (LA4_0=='$'||(LA4_0 >= '0' && LA4_0 <= '9')||(LA4_0 >= 'A' && LA4_0 <= 'Z')||LA4_0=='_'||(LA4_0 >= 'a' && LA4_0 <= 'z')) ) { + alt4=1; + } + + switch (alt4) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g: + { + if ( input.LA(1)=='$'||(input.LA(1) >= '0' && input.LA(1) <= '9')||(input.LA(1) >= 'A' && input.LA(1) <= 'Z')||input.LA(1)=='_'||(input.LA(1) >= 'a' && input.LA(1) <= 'z') ) { + input.consume(); + } + else { + MismatchedSetException mse = new MismatchedSetException(null,input); + recover(mse); + throw mse; + } + } + break; + + default : + break loop4; + } + } + + } + + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "ID" + + // $ANTLR start "STRING" + public final void mSTRING() throws RecognitionException { + try { + // src/java/org/apache/lucene/expressions/js/Javascript.g:356:5: ( '\\'' ( SINGLE_STRING_CHAR )* '\\'' | '\"' ( DOUBLE_STRING_CHAR )* '\"' ) + int alt7=2; + int LA7_0 = input.LA(1); + if ( (LA7_0=='\'') ) { + alt7=1; + } + else if ( (LA7_0=='\"') ) { + alt7=2; + } + + else { + NoViableAltException nvae = + new NoViableAltException("", 7, 0, input); + throw nvae; + } + + switch (alt7) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:356:7: '\\'' ( SINGLE_STRING_CHAR )* '\\'' + { + match('\''); + // src/java/org/apache/lucene/expressions/js/Javascript.g:356:12: ( SINGLE_STRING_CHAR )* + loop5: + while (true) { + int alt5=2; + int LA5_0 = input.LA(1); + if ( ((LA5_0 >= '\u0000' && LA5_0 <= '&')||(LA5_0 >= '(' && LA5_0 <= '\uFFFF')) ) { + alt5=1; + } + + switch (alt5) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:356:12: SINGLE_STRING_CHAR + { + mSINGLE_STRING_CHAR(); + + } + break; + + default : + break loop5; + } + } + + match('\''); + + } + break; + case 2 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:357:7: '\"' ( DOUBLE_STRING_CHAR )* '\"' + { + match('\"'); + // src/java/org/apache/lucene/expressions/js/Javascript.g:357:11: ( DOUBLE_STRING_CHAR )* + loop6: + while (true) { + int alt6=2; + int LA6_0 = input.LA(1); + if ( ((LA6_0 >= '\u0000' && LA6_0 <= '!')||(LA6_0 >= '#' && LA6_0 <= '\uFFFF')) ) { + alt6=1; + } + + switch (alt6) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:357:11: DOUBLE_STRING_CHAR + { + mDOUBLE_STRING_CHAR(); + + } + break; + + default : + break loop6; + } + } + + match('\"'); + } + break; + + } + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "STRING" + + // $ANTLR start "SINGLE_STRING_CHAR" + public final void mSINGLE_STRING_CHAR() throws RecognitionException { + try { + // src/java/org/apache/lucene/expressions/js/Javascript.g:362:5: ( '\\\\\\'' | '\\\\\\\\' |~ ( '\\\\' | '\\'' ) ) + int alt8=3; + int LA8_0 = input.LA(1); + if ( (LA8_0=='\\') ) { + int LA8_1 = input.LA(2); + if ( (LA8_1=='\'') ) { + alt8=1; + } + else if ( (LA8_1=='\\') ) { + alt8=2; + } + + else { + int nvaeMark = input.mark(); + try { + input.consume(); + NoViableAltException nvae = + new NoViableAltException("", 8, 1, input); + throw nvae; + } finally { + input.rewind(nvaeMark); + } + } + + } + else if ( ((LA8_0 >= '\u0000' && LA8_0 <= '&')||(LA8_0 >= '(' && LA8_0 <= '[')||(LA8_0 >= ']' && LA8_0 <= '\uFFFF')) ) { + alt8=3; + } + + else { + NoViableAltException nvae = + new NoViableAltException("", 8, 0, input); + throw nvae; + } + + switch (alt8) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:362:7: '\\\\\\'' + { + match("\\'"); + + } + break; + case 2 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:363:7: '\\\\\\\\' + { + match("\\\\"); + + } + break; + case 3 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:364:7: ~ ( '\\\\' | '\\'' ) + { + if ( (input.LA(1) >= '\u0000' && input.LA(1) <= '&')||(input.LA(1) >= '(' && input.LA(1) <= '[')||(input.LA(1) >= ']' && input.LA(1) <= '\uFFFF') ) { + input.consume(); + } + else { + MismatchedSetException mse = new MismatchedSetException(null,input); + recover(mse); + throw mse; + } + } + break; + + } + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "SINGLE_STRING_CHAR" + + // $ANTLR start "DOUBLE_STRING_CHAR" + public final void mDOUBLE_STRING_CHAR() throws RecognitionException { + try { + // src/java/org/apache/lucene/expressions/js/Javascript.g:369:5: ( '\\\\\"' | '\\\\\\\\' |~ ( '\\\\' | '\"' ) ) + int alt9=3; + int LA9_0 = input.LA(1); + if ( (LA9_0=='\\') ) { + int LA9_1 = input.LA(2); + if ( (LA9_1=='\"') ) { + alt9=1; + } + else if ( (LA9_1=='\\') ) { + alt9=2; + } + + else { + int nvaeMark = input.mark(); + try { + input.consume(); + NoViableAltException nvae = + new NoViableAltException("", 9, 1, input); + throw nvae; + } finally { + input.rewind(nvaeMark); + } + } + + } + else if ( ((LA9_0 >= '\u0000' && LA9_0 <= '!')||(LA9_0 >= '#' && LA9_0 <= '[')||(LA9_0 >= ']' && LA9_0 <= '\uFFFF')) ) { + alt9=3; + } + + else { + NoViableAltException nvae = + new NoViableAltException("", 9, 0, input); + throw nvae; + } + + switch (alt9) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:369:7: '\\\\\"' + { + match("\\\""); + + } + break; + case 2 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:370:7: '\\\\\\\\' + { + match("\\\\"); + + } + break; + case 3 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:371:7: ~ ( '\\\\' | '\"' ) + { + if ( (input.LA(1) >= '\u0000' && input.LA(1) <= '!')||(input.LA(1) >= '#' && input.LA(1) <= '[')||(input.LA(1) >= ']' && input.LA(1) <= '\uFFFF') ) { + input.consume(); + } + else { + MismatchedSetException mse = new MismatchedSetException(null,input); + recover(mse); + throw mse; + } + } + break; + + } + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "DOUBLE_STRING_CHAR" + + // $ANTLR start "WS" + public final void mWS() throws RecognitionException { + try { + int _type = WS; + int _channel = DEFAULT_TOKEN_CHANNEL; + // src/java/org/apache/lucene/expressions/js/Javascript.g:374:5: ( ( ' ' | '\\t' | '\\n' | '\\r' )+ ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:374:7: ( ' ' | '\\t' | '\\n' | '\\r' )+ + { + // src/java/org/apache/lucene/expressions/js/Javascript.g:374:7: ( ' ' | '\\t' | '\\n' | '\\r' )+ + int cnt10=0; + loop10: + while (true) { + int alt10=2; + int LA10_0 = input.LA(1); + if ( ((LA10_0 >= '\t' && LA10_0 <= '\n')||LA10_0=='\r'||LA10_0==' ') ) { + alt10=1; + } + + switch (alt10) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g: + { + if ( (input.LA(1) >= '\t' && input.LA(1) <= '\n')||input.LA(1)=='\r'||input.LA(1)==' ' ) { + input.consume(); + } + else { + MismatchedSetException mse = new MismatchedSetException(null,input); + recover(mse); + throw mse; + } + } + break; + + default : + if ( cnt10 >= 1 ) break loop10; + EarlyExitException eee = new EarlyExitException(10, input); + throw eee; + } + cnt10++; + } + + skip(); + } + + state.type = _type; + state.channel = _channel; + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "WS" + + // $ANTLR start "DECIMAL" + public final void mDECIMAL() throws RecognitionException { + try { + int _type = DECIMAL; + int _channel = DEFAULT_TOKEN_CHANNEL; + // src/java/org/apache/lucene/expressions/js/Javascript.g:378:5: ( DECIMALINTEGER AT_DOT ( DECIMALDIGIT )* ( EXPONENT )? | AT_DOT ( DECIMALDIGIT )+ ( EXPONENT )? | DECIMALINTEGER ( EXPONENT )? ) + int alt16=3; + alt16 = dfa16.predict(input); + switch (alt16) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:378:7: DECIMALINTEGER AT_DOT ( DECIMALDIGIT )* ( EXPONENT )? + { + mDECIMALINTEGER(); + + mAT_DOT(); + + // src/java/org/apache/lucene/expressions/js/Javascript.g:378:29: ( DECIMALDIGIT )* + loop11: + while (true) { + int alt11=2; + int LA11_0 = input.LA(1); + if ( ((LA11_0 >= '0' && LA11_0 <= '9')) ) { + alt11=1; + } + + switch (alt11) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g: + { + if ( (input.LA(1) >= '0' && input.LA(1) <= '9') ) { + input.consume(); + } + else { + MismatchedSetException mse = new MismatchedSetException(null,input); + recover(mse); + throw mse; + } + } + break; + + default : + break loop11; + } + } + + // src/java/org/apache/lucene/expressions/js/Javascript.g:378:43: ( EXPONENT )? + int alt12=2; + int LA12_0 = input.LA(1); + if ( (LA12_0=='E'||LA12_0=='e') ) { + alt12=1; + } + switch (alt12) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:378:43: EXPONENT + { + mEXPONENT(); + + } + break; + + } + + } + break; + case 2 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:379:7: AT_DOT ( DECIMALDIGIT )+ ( EXPONENT )? + { + mAT_DOT(); + + // src/java/org/apache/lucene/expressions/js/Javascript.g:379:14: ( DECIMALDIGIT )+ + int cnt13=0; + loop13: + while (true) { + int alt13=2; + int LA13_0 = input.LA(1); + if ( ((LA13_0 >= '0' && LA13_0 <= '9')) ) { + alt13=1; + } + + switch (alt13) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g: + { + if ( (input.LA(1) >= '0' && input.LA(1) <= '9') ) { + input.consume(); + } + else { + MismatchedSetException mse = new MismatchedSetException(null,input); + recover(mse); + throw mse; + } + } + break; + + default : + if ( cnt13 >= 1 ) break loop13; + EarlyExitException eee = new EarlyExitException(13, input); + throw eee; + } + cnt13++; + } + + // src/java/org/apache/lucene/expressions/js/Javascript.g:379:28: ( EXPONENT )? + int alt14=2; + int LA14_0 = input.LA(1); + if ( (LA14_0=='E'||LA14_0=='e') ) { + alt14=1; + } + switch (alt14) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:379:28: EXPONENT + { + mEXPONENT(); + + } + break; + + } + + } + break; + case 3 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:380:7: DECIMALINTEGER ( EXPONENT )? + { + mDECIMALINTEGER(); + + // src/java/org/apache/lucene/expressions/js/Javascript.g:380:22: ( EXPONENT )? + int alt15=2; + int LA15_0 = input.LA(1); + if ( (LA15_0=='E'||LA15_0=='e') ) { + alt15=1; + } + switch (alt15) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:380:22: EXPONENT + { + mEXPONENT(); + + } + break; + + } + + } + break; + + } + state.type = _type; + state.channel = _channel; + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "DECIMAL" + + // $ANTLR start "OCTAL" + public final void mOCTAL() throws RecognitionException { + try { + int _type = OCTAL; + int _channel = DEFAULT_TOKEN_CHANNEL; + // src/java/org/apache/lucene/expressions/js/Javascript.g:384:5: ( '0' ( OCTALDIGIT )+ ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:384:7: '0' ( OCTALDIGIT )+ + { + match('0'); + // src/java/org/apache/lucene/expressions/js/Javascript.g:384:11: ( OCTALDIGIT )+ + int cnt17=0; + loop17: + while (true) { + int alt17=2; + int LA17_0 = input.LA(1); + if ( ((LA17_0 >= '0' && LA17_0 <= '7')) ) { + alt17=1; + } + + switch (alt17) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g: + { + if ( (input.LA(1) >= '0' && input.LA(1) <= '7') ) { + input.consume(); + } + else { + MismatchedSetException mse = new MismatchedSetException(null,input); + recover(mse); + throw mse; + } + } + break; + + default : + if ( cnt17 >= 1 ) break loop17; + EarlyExitException eee = new EarlyExitException(17, input); + throw eee; + } + cnt17++; + } + + } + + state.type = _type; + state.channel = _channel; + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "OCTAL" + + // $ANTLR start "HEX" + public final void mHEX() throws RecognitionException { + try { + int _type = HEX; + int _channel = DEFAULT_TOKEN_CHANNEL; + // src/java/org/apache/lucene/expressions/js/Javascript.g:388:5: ( ( '0x' | '0X' ) ( HEXDIGIT )+ ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:388:7: ( '0x' | '0X' ) ( HEXDIGIT )+ + { + // src/java/org/apache/lucene/expressions/js/Javascript.g:388:7: ( '0x' | '0X' ) + int alt18=2; + int LA18_0 = input.LA(1); + if ( (LA18_0=='0') ) { + int LA18_1 = input.LA(2); + if ( (LA18_1=='x') ) { + alt18=1; + } + else if ( (LA18_1=='X') ) { + alt18=2; + } + + else { + int nvaeMark = input.mark(); + try { + input.consume(); + NoViableAltException nvae = + new NoViableAltException("", 18, 1, input); + throw nvae; + } finally { + input.rewind(nvaeMark); + } + } + + } + + else { + NoViableAltException nvae = + new NoViableAltException("", 18, 0, input); + throw nvae; + } + + switch (alt18) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:388:8: '0x' + { + match("0x"); + + } + break; + case 2 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:388:13: '0X' + { + match("0X"); + + } + break; + + } + + // src/java/org/apache/lucene/expressions/js/Javascript.g:388:19: ( HEXDIGIT )+ + int cnt19=0; + loop19: + while (true) { + int alt19=2; + int LA19_0 = input.LA(1); + if ( ((LA19_0 >= '0' && LA19_0 <= '9')||(LA19_0 >= 'A' && LA19_0 <= 'F')||(LA19_0 >= 'a' && LA19_0 <= 'f')) ) { + alt19=1; + } + + switch (alt19) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g: + { + if ( (input.LA(1) >= '0' && input.LA(1) <= '9')||(input.LA(1) >= 'A' && input.LA(1) <= 'F')||(input.LA(1) >= 'a' && input.LA(1) <= 'f') ) { + input.consume(); + } + else { + MismatchedSetException mse = new MismatchedSetException(null,input); + recover(mse); + throw mse; + } + } + break; + + default : + if ( cnt19 >= 1 ) break loop19; + EarlyExitException eee = new EarlyExitException(19, input); + throw eee; + } + cnt19++; + } + + } + + state.type = _type; + state.channel = _channel; + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "HEX" + + // $ANTLR start "DECIMALINTEGER" + public final void mDECIMALINTEGER() throws RecognitionException { + try { + // src/java/org/apache/lucene/expressions/js/Javascript.g:394:5: ( '0' | '1' .. '9' ( DECIMALDIGIT )* ) + int alt21=2; + int LA21_0 = input.LA(1); + if ( (LA21_0=='0') ) { + alt21=1; + } + else if ( ((LA21_0 >= '1' && LA21_0 <= '9')) ) { + alt21=2; + } + + else { + NoViableAltException nvae = + new NoViableAltException("", 21, 0, input); + throw nvae; + } + + switch (alt21) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:394:7: '0' + { + match('0'); + } + break; + case 2 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:395:7: '1' .. '9' ( DECIMALDIGIT )* + { + matchRange('1','9'); + // src/java/org/apache/lucene/expressions/js/Javascript.g:395:16: ( DECIMALDIGIT )* + loop20: + while (true) { + int alt20=2; + int LA20_0 = input.LA(1); + if ( ((LA20_0 >= '0' && LA20_0 <= '9')) ) { + alt20=1; + } + + switch (alt20) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g: + { + if ( (input.LA(1) >= '0' && input.LA(1) <= '9') ) { + input.consume(); + } + else { + MismatchedSetException mse = new MismatchedSetException(null,input); + recover(mse); + throw mse; + } + } + break; + + default : + break loop20; + } + } + + } + break; + + } + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "DECIMALINTEGER" + + // $ANTLR start "EXPONENT" + public final void mEXPONENT() throws RecognitionException { + try { + // src/java/org/apache/lucene/expressions/js/Javascript.g:400:5: ( ( 'e' | 'E' ) ( '+' | '-' )? ( DECIMALDIGIT )+ ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:400:7: ( 'e' | 'E' ) ( '+' | '-' )? ( DECIMALDIGIT )+ + { + if ( input.LA(1)=='E'||input.LA(1)=='e' ) { + input.consume(); + } + else { + MismatchedSetException mse = new MismatchedSetException(null,input); + recover(mse); + throw mse; + } + // src/java/org/apache/lucene/expressions/js/Javascript.g:400:17: ( '+' | '-' )? + int alt22=2; + int LA22_0 = input.LA(1); + if ( (LA22_0=='+'||LA22_0=='-') ) { + alt22=1; + } + switch (alt22) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g: + { + if ( input.LA(1)=='+'||input.LA(1)=='-' ) { + input.consume(); + } + else { + MismatchedSetException mse = new MismatchedSetException(null,input); + recover(mse); + throw mse; + } + } + break; + + } + + // src/java/org/apache/lucene/expressions/js/Javascript.g:400:28: ( DECIMALDIGIT )+ + int cnt23=0; + loop23: + while (true) { + int alt23=2; + int LA23_0 = input.LA(1); + if ( ((LA23_0 >= '0' && LA23_0 <= '9')) ) { + alt23=1; + } + + switch (alt23) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g: + { + if ( (input.LA(1) >= '0' && input.LA(1) <= '9') ) { + input.consume(); + } + else { + MismatchedSetException mse = new MismatchedSetException(null,input); + recover(mse); + throw mse; + } + } + break; + + default : + if ( cnt23 >= 1 ) break loop23; + EarlyExitException eee = new EarlyExitException(23, input); + throw eee; + } + cnt23++; + } + + } + + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "EXPONENT" + + // $ANTLR start "DECIMALDIGIT" + public final void mDECIMALDIGIT() throws RecognitionException { + try { + // src/java/org/apache/lucene/expressions/js/Javascript.g:405:5: ( '0' .. '9' ) + // src/java/org/apache/lucene/expressions/js/Javascript.g: + { + if ( (input.LA(1) >= '0' && input.LA(1) <= '9') ) { + input.consume(); + } + else { + MismatchedSetException mse = new MismatchedSetException(null,input); + recover(mse); + throw mse; + } + } + + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "DECIMALDIGIT" + + // $ANTLR start "HEXDIGIT" + public final void mHEXDIGIT() throws RecognitionException { + try { + // src/java/org/apache/lucene/expressions/js/Javascript.g:410:5: ( DECIMALDIGIT | 'a' .. 'f' | 'A' .. 'F' ) + // src/java/org/apache/lucene/expressions/js/Javascript.g: + { + if ( (input.LA(1) >= '0' && input.LA(1) <= '9')||(input.LA(1) >= 'A' && input.LA(1) <= 'F')||(input.LA(1) >= 'a' && input.LA(1) <= 'f') ) { + input.consume(); + } + else { + MismatchedSetException mse = new MismatchedSetException(null,input); + recover(mse); + throw mse; + } + } + + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "HEXDIGIT" + + // $ANTLR start "OCTALDIGIT" + public final void mOCTALDIGIT() throws RecognitionException { + try { + // src/java/org/apache/lucene/expressions/js/Javascript.g:417:5: ( '0' .. '7' ) + // src/java/org/apache/lucene/expressions/js/Javascript.g: + { + if ( (input.LA(1) >= '0' && input.LA(1) <= '7') ) { + input.consume(); + } + else { + MismatchedSetException mse = new MismatchedSetException(null,input); + recover(mse); + throw mse; + } + } + + } + finally { + // do for sure before leaving + } + } + // $ANTLR end "OCTALDIGIT" + + @Override + public void mTokens() throws RecognitionException { + // src/java/org/apache/lucene/expressions/js/Javascript.g:1:8: ( AT_ADD | AT_BIT_AND | AT_BIT_NOT | AT_BIT_OR | AT_BIT_SHL | AT_BIT_SHR | AT_BIT_SHU | AT_BIT_XOR | AT_BOOL_AND | AT_BOOL_NOT | AT_BOOL_OR | AT_COLON | AT_COMMA | AT_COMP_EQ | AT_COMP_GT | AT_COMP_GTE | AT_COMP_LT | AT_COMP_LTE | AT_COMP_NEQ | AT_COND_QUE | AT_DIVIDE | AT_DOT | AT_LPAREN | AT_MODULO | AT_MULTIPLY | AT_RPAREN | AT_SUBTRACT | VARIABLE | WS | DECIMAL | OCTAL | HEX ) + int alt24=32; + switch ( input.LA(1) ) { + case '+': + { + alt24=1; + } + break; + case '&': + { + int LA24_2 = input.LA(2); + if ( (LA24_2=='&') ) { + alt24=9; + } + + else { + alt24=2; + } + + } + break; + case '~': + { + alt24=3; + } + break; + case '|': + { + int LA24_4 = input.LA(2); + if ( (LA24_4=='|') ) { + alt24=11; + } + + else { + alt24=4; + } + + } + break; + case '<': + { + switch ( input.LA(2) ) { + case '<': + { + alt24=5; + } + break; + case '=': + { + alt24=18; + } + break; + default: + alt24=17; + } + } + break; + case '>': + { + switch ( input.LA(2) ) { + case '>': + { + int LA24_31 = input.LA(3); + if ( (LA24_31=='>') ) { + alt24=7; + } + + else { + alt24=6; + } + + } + break; + case '=': + { + alt24=16; + } + break; + default: + alt24=15; + } + } + break; + case '^': + { + alt24=8; + } + break; + case '!': + { + int LA24_8 = input.LA(2); + if ( (LA24_8=='=') ) { + alt24=19; + } + + else { + alt24=10; + } + + } + break; + case ':': + { + alt24=12; + } + break; + case ',': + { + alt24=13; + } + break; + case '=': + { + alt24=14; + } + break; + case '?': + { + alt24=20; + } + break; + case '/': + { + alt24=21; + } + break; + case '.': + { + int LA24_14 = input.LA(2); + if ( ((LA24_14 >= '0' && LA24_14 <= '9')) ) { + alt24=30; + } + + else { + alt24=22; + } + + } + break; + case '(': + { + alt24=23; + } + break; + case '%': + { + alt24=24; + } + break; + case '*': + { + alt24=25; + } + break; + case ')': + { + alt24=26; + } + break; + case '-': + { + alt24=27; + } + break; + case '$': + case 'A': + case 'B': + case 'C': + case 'D': + case 'E': + case 'F': + case 'G': + case 'H': + case 'I': + case 'J': + case 'K': + case 'L': + case 'M': + case 'N': + case 'O': + case 'P': + case 'Q': + case 'R': + case 'S': + case 'T': + case 'U': + case 'V': + case 'W': + case 'X': + case 'Y': + case 'Z': + case '_': + case 'a': + case 'b': + case 'c': + case 'd': + case 'e': + case 'f': + case 'g': + case 'h': + case 'i': + case 'j': + case 'k': + case 'l': + case 'm': + case 'n': + case 'o': + case 'p': + case 'q': + case 'r': + case 's': + case 't': + case 'u': + case 'v': + case 'w': + case 'x': + case 'y': + case 'z': + { + alt24=28; + } + break; + case '\t': + case '\n': + case '\r': + case ' ': + { + alt24=29; + } + break; + case '0': + { + switch ( input.LA(2) ) { + case 'X': + case 'x': + { + alt24=32; + } + break; + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + { + alt24=31; + } + break; + default: + alt24=30; + } + } + break; + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + { + alt24=30; + } + break; + default: + NoViableAltException nvae = + new NoViableAltException("", 24, 0, input); + throw nvae; + } + switch (alt24) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:1:10: AT_ADD + { + mAT_ADD(); + + } + break; + case 2 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:1:17: AT_BIT_AND + { + mAT_BIT_AND(); + + } + break; + case 3 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:1:28: AT_BIT_NOT + { + mAT_BIT_NOT(); + + } + break; + case 4 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:1:39: AT_BIT_OR + { + mAT_BIT_OR(); + + } + break; + case 5 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:1:49: AT_BIT_SHL + { + mAT_BIT_SHL(); + + } + break; + case 6 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:1:60: AT_BIT_SHR + { + mAT_BIT_SHR(); + + } + break; + case 7 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:1:71: AT_BIT_SHU + { + mAT_BIT_SHU(); + + } + break; + case 8 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:1:82: AT_BIT_XOR + { + mAT_BIT_XOR(); + + } + break; + case 9 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:1:93: AT_BOOL_AND + { + mAT_BOOL_AND(); + + } + break; + case 10 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:1:105: AT_BOOL_NOT + { + mAT_BOOL_NOT(); + + } + break; + case 11 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:1:117: AT_BOOL_OR + { + mAT_BOOL_OR(); + + } + break; + case 12 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:1:128: AT_COLON + { + mAT_COLON(); + + } + break; + case 13 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:1:137: AT_COMMA + { + mAT_COMMA(); + + } + break; + case 14 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:1:146: AT_COMP_EQ + { + mAT_COMP_EQ(); + + } + break; + case 15 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:1:157: AT_COMP_GT + { + mAT_COMP_GT(); + + } + break; + case 16 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:1:168: AT_COMP_GTE + { + mAT_COMP_GTE(); + + } + break; + case 17 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:1:180: AT_COMP_LT + { + mAT_COMP_LT(); + + } + break; + case 18 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:1:191: AT_COMP_LTE + { + mAT_COMP_LTE(); + + } + break; + case 19 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:1:203: AT_COMP_NEQ + { + mAT_COMP_NEQ(); + + } + break; + case 20 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:1:215: AT_COND_QUE + { + mAT_COND_QUE(); + + } + break; + case 21 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:1:227: AT_DIVIDE + { + mAT_DIVIDE(); + + } + break; + case 22 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:1:237: AT_DOT + { + mAT_DOT(); + + } + break; + case 23 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:1:244: AT_LPAREN + { + mAT_LPAREN(); + + } + break; + case 24 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:1:254: AT_MODULO + { + mAT_MODULO(); + + } + break; + case 25 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:1:264: AT_MULTIPLY + { + mAT_MULTIPLY(); + + } + break; + case 26 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:1:276: AT_RPAREN + { + mAT_RPAREN(); + + } + break; + case 27 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:1:286: AT_SUBTRACT + { + mAT_SUBTRACT(); + + } + break; + case 28 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:1:298: VARIABLE + { + mVARIABLE(); + + } + break; + case 29 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:1:307: WS + { + mWS(); + + } + break; + case 30 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:1:310: DECIMAL + { + mDECIMAL(); + + } + break; + case 31 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:1:318: OCTAL + { + mOCTAL(); + + } + break; + case 32 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:1:324: HEX + { + mHEX(); + + } + break; + + } + } + + + protected DFA16 dfa16 = new DFA16(this); + static final String DFA16_eotS = + "\1\uffff\2\4\3\uffff\1\4"; + static final String DFA16_eofS = + "\7\uffff"; + static final String DFA16_minS = + "\3\56\3\uffff\1\56"; + static final String DFA16_maxS = + "\1\71\1\56\1\71\3\uffff\1\71"; + static final String DFA16_acceptS = + "\3\uffff\1\2\1\3\1\1\1\uffff"; + static final String DFA16_specialS = + "\7\uffff}>"; + static final String[] DFA16_transitionS = { + "\1\3\1\uffff\1\1\11\2", + "\1\5", + "\1\5\1\uffff\12\6", + "", + "", + "", + "\1\5\1\uffff\12\6" + }; + + static final short[] DFA16_eot = DFA.unpackEncodedString(DFA16_eotS); + static final short[] DFA16_eof = DFA.unpackEncodedString(DFA16_eofS); + static final char[] DFA16_min = DFA.unpackEncodedStringToUnsignedChars(DFA16_minS); + static final char[] DFA16_max = DFA.unpackEncodedStringToUnsignedChars(DFA16_maxS); + static final short[] DFA16_accept = DFA.unpackEncodedString(DFA16_acceptS); + static final short[] DFA16_special = DFA.unpackEncodedString(DFA16_specialS); + static final short[][] DFA16_transition; + + static { + int numStates = DFA16_transitionS.length; + DFA16_transition = new short[numStates][]; + for (int i=0; i", "", "", "", "ARRAY", "AT_ADD", "AT_BIT_AND", + "AT_BIT_NOT", "AT_BIT_OR", "AT_BIT_SHL", "AT_BIT_SHR", "AT_BIT_SHU", "AT_BIT_XOR", + "AT_BOOL_AND", "AT_BOOL_NOT", "AT_BOOL_OR", "AT_CALL", "AT_COLON", "AT_COMMA", + "AT_COMP_EQ", "AT_COMP_GT", "AT_COMP_GTE", "AT_COMP_LT", "AT_COMP_LTE", + "AT_COMP_NEQ", "AT_COND_QUE", "AT_DIVIDE", "AT_DOT", "AT_LPAREN", "AT_MODULO", + "AT_MULTIPLY", "AT_NEGATE", "AT_RPAREN", "AT_SUBTRACT", "DECIMAL", "DECIMALDIGIT", + "DECIMALINTEGER", "DOUBLE_STRING_CHAR", "EXPONENT", "HEX", "HEXDIGIT", + "ID", "OBJECT", "OCTAL", "OCTALDIGIT", "SINGLE_STRING_CHAR", "STRING", + "VARIABLE", "WS" + }; + public static final int EOF=-1; + public static final int ARRAY=4; + public static final int AT_ADD=5; + public static final int AT_BIT_AND=6; + public static final int AT_BIT_NOT=7; + public static final int AT_BIT_OR=8; + public static final int AT_BIT_SHL=9; + public static final int AT_BIT_SHR=10; + public static final int AT_BIT_SHU=11; + public static final int AT_BIT_XOR=12; + public static final int AT_BOOL_AND=13; + public static final int AT_BOOL_NOT=14; + public static final int AT_BOOL_OR=15; + public static final int AT_CALL=16; + public static final int AT_COLON=17; + public static final int AT_COMMA=18; + public static final int AT_COMP_EQ=19; + public static final int AT_COMP_GT=20; + public static final int AT_COMP_GTE=21; + public static final int AT_COMP_LT=22; + public static final int AT_COMP_LTE=23; + public static final int AT_COMP_NEQ=24; + public static final int AT_COND_QUE=25; + public static final int AT_DIVIDE=26; + public static final int AT_DOT=27; + public static final int AT_LPAREN=28; + public static final int AT_MODULO=29; + public static final int AT_MULTIPLY=30; + public static final int AT_NEGATE=31; + public static final int AT_RPAREN=32; + public static final int AT_SUBTRACT=33; + public static final int DECIMAL=34; + public static final int DECIMALDIGIT=35; + public static final int DECIMALINTEGER=36; + public static final int DOUBLE_STRING_CHAR=37; + public static final int EXPONENT=38; + public static final int HEX=39; + public static final int HEXDIGIT=40; + public static final int ID=41; + public static final int OBJECT=42; + public static final int OCTAL=43; + public static final int OCTALDIGIT=44; + public static final int SINGLE_STRING_CHAR=45; + public static final int STRING=46; + public static final int VARIABLE=47; + public static final int WS=48; + + // delegates + public Parser[] getDelegates() { + return new Parser[] {}; + } + + // delegators + + + public XJavascriptParser(TokenStream input) { + this(input, new RecognizerSharedState()); + } + public XJavascriptParser(TokenStream input, RecognizerSharedState state) { + super(input, state); + } + + protected TreeAdaptor adaptor = new CommonTreeAdaptor(); + + public void setTreeAdaptor(TreeAdaptor adaptor) { + this.adaptor = adaptor; + } + public TreeAdaptor getTreeAdaptor() { + return adaptor; + } + @Override public String[] getTokenNames() { return XJavascriptParser.tokenNames; } + @Override public String getGrammarFileName() { return "src/java/org/apache/lucene/expressions/js/Javascript.g"; } + + + + @Override + public void displayRecognitionError(String[] tokenNames, RecognitionException re) { + String message; + + if (re.token == null) { + message = " unknown error (missing token)."; + } + else if (re instanceof UnwantedTokenException) { + message = " extraneous " + getReadableTokenString(re.token) + + " at position (" + re.charPositionInLine + ")."; + } + else if (re instanceof MissingTokenException) { + message = " missing " + getReadableTokenString(re.token) + + " at position (" + re.charPositionInLine + ")."; + } + else if (re instanceof NoViableAltException) { + switch (re.token.getType()) { + case EOF: + message = " unexpected end of expression."; + break; + default: + message = " invalid sequence of tokens near " + getReadableTokenString(re.token) + + " at position (" + re.charPositionInLine + ")."; + break; + } + } + else { + message = " unexpected token " + getReadableTokenString(re.token) + + " at position (" + re.charPositionInLine + ")."; + } + ParseException parseException = new ParseException(message, re.charPositionInLine); + parseException.initCause(re); + throw new RuntimeException(parseException); + } + + public static String getReadableTokenString(Token token) { + if (token == null) { + return "unknown token"; + } + + switch (token.getType()) { + case AT_LPAREN: + return "open parenthesis '('"; + case AT_RPAREN: + return "close parenthesis ')'"; + case AT_COMP_LT: + return "less than '<'"; + case AT_COMP_LTE: + return "less than or equal '<='"; + case AT_COMP_GT: + return "greater than '>'"; + case AT_COMP_GTE: + return "greater than or equal '>='"; + case AT_COMP_EQ: + return "equal '=='"; + case AT_NEGATE: + return "negate '!='"; + case AT_BOOL_NOT: + return "boolean not '!'"; + case AT_BOOL_AND: + return "boolean and '&&'"; + case AT_BOOL_OR: + return "boolean or '||'"; + case AT_COND_QUE: + return "conditional '?'"; + case AT_ADD: + return "addition '+'"; + case AT_SUBTRACT: + return "subtraction '-'"; + case AT_MULTIPLY: + return "multiplication '*'"; + case AT_DIVIDE: + return "division '/'"; + case AT_MODULO: + return "modulo '%'"; + case AT_BIT_SHL: + return "bit shift left '<<'"; + case AT_BIT_SHR: + return "bit shift right '>>'"; + case AT_BIT_SHU: + return "unsigned bit shift right '>>>'"; + case AT_BIT_AND: + return "bitwise and '&'"; + case AT_BIT_OR: + return "bitwise or '|'"; + case AT_BIT_XOR: + return "bitwise xor '^'"; + case AT_BIT_NOT: + return "bitwise not '~'"; + case ID: + return "identifier '" + token.getText() + "'"; + case DECIMAL: + return "decimal '" + token.getText() + "'"; + case OCTAL: + return "octal '" + token.getText() + "'"; + case HEX: + return "hex '" + token.getText() + "'"; + case EOF: + return "end of expression"; + default: + return "'" + token.getText() + "'"; + } + } + + + + public static class expression_return extends ParserRuleReturnScope { + CommonTree tree; + @Override + public CommonTree getTree() { return tree; } + }; + + + // $ANTLR start "expression" + // src/java/org/apache/lucene/expressions/js/Javascript.g:250:1: expression : conditional EOF !; + public final XJavascriptParser.expression_return expression() throws RecognitionException { + XJavascriptParser.expression_return retval = new XJavascriptParser.expression_return(); + retval.start = input.LT(1); + + CommonTree root_0 = null; + + Token EOF2=null; + ParserRuleReturnScope conditional1 =null; + + CommonTree EOF2_tree=null; + + try { + // src/java/org/apache/lucene/expressions/js/Javascript.g:251:5: ( conditional EOF !) + // src/java/org/apache/lucene/expressions/js/Javascript.g:251:7: conditional EOF ! + { + root_0 = (CommonTree)adaptor.nil(); + + + pushFollow(FOLLOW_conditional_in_expression737); + conditional1=conditional(); + state._fsp--; + + adaptor.addChild(root_0, conditional1.getTree()); + + EOF2=(Token)match(input,EOF,FOLLOW_EOF_in_expression739); + } + + retval.stop = input.LT(-1); + + retval.tree = (CommonTree)adaptor.rulePostProcessing(root_0); + adaptor.setTokenBoundaries(retval.tree, retval.start, retval.stop); + + } + catch (RecognitionException re) { + reportError(re); + recover(input,re); + retval.tree = (CommonTree)adaptor.errorNode(input, retval.start, input.LT(-1), re); + } + finally { + // do for sure before leaving + } + return retval; + } + // $ANTLR end "expression" + + + public static class conditional_return extends ParserRuleReturnScope { + CommonTree tree; + @Override + public CommonTree getTree() { return tree; } + }; + + + // $ANTLR start "conditional" + // src/java/org/apache/lucene/expressions/js/Javascript.g:254:1: conditional : logical_or ( AT_COND_QUE ^ conditional AT_COLON ! conditional )? ; + public final XJavascriptParser.conditional_return conditional() throws RecognitionException { + XJavascriptParser.conditional_return retval = new XJavascriptParser.conditional_return(); + retval.start = input.LT(1); + + CommonTree root_0 = null; + + Token AT_COND_QUE4=null; + Token AT_COLON6=null; + ParserRuleReturnScope logical_or3 =null; + ParserRuleReturnScope conditional5 =null; + ParserRuleReturnScope conditional7 =null; + + CommonTree AT_COND_QUE4_tree=null; + CommonTree AT_COLON6_tree=null; + + try { + // src/java/org/apache/lucene/expressions/js/Javascript.g:255:5: ( logical_or ( AT_COND_QUE ^ conditional AT_COLON ! conditional )? ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:255:7: logical_or ( AT_COND_QUE ^ conditional AT_COLON ! conditional )? + { + root_0 = (CommonTree)adaptor.nil(); + + + pushFollow(FOLLOW_logical_or_in_conditional757); + logical_or3=logical_or(); + state._fsp--; + + adaptor.addChild(root_0, logical_or3.getTree()); + + // src/java/org/apache/lucene/expressions/js/Javascript.g:255:18: ( AT_COND_QUE ^ conditional AT_COLON ! conditional )? + int alt1=2; + int LA1_0 = input.LA(1); + if ( (LA1_0==AT_COND_QUE) ) { + alt1=1; + } + switch (alt1) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:255:19: AT_COND_QUE ^ conditional AT_COLON ! conditional + { + AT_COND_QUE4=(Token)match(input,AT_COND_QUE,FOLLOW_AT_COND_QUE_in_conditional760); + AT_COND_QUE4_tree = (CommonTree)adaptor.create(AT_COND_QUE4); + root_0 = (CommonTree)adaptor.becomeRoot(AT_COND_QUE4_tree, root_0); + + pushFollow(FOLLOW_conditional_in_conditional763); + conditional5=conditional(); + state._fsp--; + + adaptor.addChild(root_0, conditional5.getTree()); + + AT_COLON6=(Token)match(input,AT_COLON,FOLLOW_AT_COLON_in_conditional765); + pushFollow(FOLLOW_conditional_in_conditional768); + conditional7=conditional(); + state._fsp--; + + adaptor.addChild(root_0, conditional7.getTree()); + + } + break; + + } + + } + + retval.stop = input.LT(-1); + + retval.tree = (CommonTree)adaptor.rulePostProcessing(root_0); + adaptor.setTokenBoundaries(retval.tree, retval.start, retval.stop); + + } + catch (RecognitionException re) { + reportError(re); + recover(input,re); + retval.tree = (CommonTree)adaptor.errorNode(input, retval.start, input.LT(-1), re); + } + finally { + // do for sure before leaving + } + return retval; + } + // $ANTLR end "conditional" + + + public static class logical_or_return extends ParserRuleReturnScope { + CommonTree tree; + @Override + public CommonTree getTree() { return tree; } + }; + + + // $ANTLR start "logical_or" + // src/java/org/apache/lucene/expressions/js/Javascript.g:258:1: logical_or : logical_and ( AT_BOOL_OR ^ logical_and )* ; + public final XJavascriptParser.logical_or_return logical_or() throws RecognitionException { + XJavascriptParser.logical_or_return retval = new XJavascriptParser.logical_or_return(); + retval.start = input.LT(1); + + CommonTree root_0 = null; + + Token AT_BOOL_OR9=null; + ParserRuleReturnScope logical_and8 =null; + ParserRuleReturnScope logical_and10 =null; + + CommonTree AT_BOOL_OR9_tree=null; + + try { + // src/java/org/apache/lucene/expressions/js/Javascript.g:259:5: ( logical_and ( AT_BOOL_OR ^ logical_and )* ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:259:7: logical_and ( AT_BOOL_OR ^ logical_and )* + { + root_0 = (CommonTree)adaptor.nil(); + + + pushFollow(FOLLOW_logical_and_in_logical_or787); + logical_and8=logical_and(); + state._fsp--; + + adaptor.addChild(root_0, logical_and8.getTree()); + + // src/java/org/apache/lucene/expressions/js/Javascript.g:259:19: ( AT_BOOL_OR ^ logical_and )* + loop2: + while (true) { + int alt2=2; + int LA2_0 = input.LA(1); + if ( (LA2_0==AT_BOOL_OR) ) { + alt2=1; + } + + switch (alt2) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:259:20: AT_BOOL_OR ^ logical_and + { + AT_BOOL_OR9=(Token)match(input,AT_BOOL_OR,FOLLOW_AT_BOOL_OR_in_logical_or790); + AT_BOOL_OR9_tree = (CommonTree)adaptor.create(AT_BOOL_OR9); + root_0 = (CommonTree)adaptor.becomeRoot(AT_BOOL_OR9_tree, root_0); + + pushFollow(FOLLOW_logical_and_in_logical_or793); + logical_and10=logical_and(); + state._fsp--; + + adaptor.addChild(root_0, logical_and10.getTree()); + + } + break; + + default : + break loop2; + } + } + + } + + retval.stop = input.LT(-1); + + retval.tree = (CommonTree)adaptor.rulePostProcessing(root_0); + adaptor.setTokenBoundaries(retval.tree, retval.start, retval.stop); + + } + catch (RecognitionException re) { + reportError(re); + recover(input,re); + retval.tree = (CommonTree)adaptor.errorNode(input, retval.start, input.LT(-1), re); + } + finally { + // do for sure before leaving + } + return retval; + } + // $ANTLR end "logical_or" + + + public static class logical_and_return extends ParserRuleReturnScope { + CommonTree tree; + @Override + public CommonTree getTree() { return tree; } + }; + + + // $ANTLR start "logical_and" + // src/java/org/apache/lucene/expressions/js/Javascript.g:262:1: logical_and : bitwise_or ( AT_BOOL_AND ^ bitwise_or )* ; + public final XJavascriptParser.logical_and_return logical_and() throws RecognitionException { + XJavascriptParser.logical_and_return retval = new XJavascriptParser.logical_and_return(); + retval.start = input.LT(1); + + CommonTree root_0 = null; + + Token AT_BOOL_AND12=null; + ParserRuleReturnScope bitwise_or11 =null; + ParserRuleReturnScope bitwise_or13 =null; + + CommonTree AT_BOOL_AND12_tree=null; + + try { + // src/java/org/apache/lucene/expressions/js/Javascript.g:263:5: ( bitwise_or ( AT_BOOL_AND ^ bitwise_or )* ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:263:7: bitwise_or ( AT_BOOL_AND ^ bitwise_or )* + { + root_0 = (CommonTree)adaptor.nil(); + + + pushFollow(FOLLOW_bitwise_or_in_logical_and812); + bitwise_or11=bitwise_or(); + state._fsp--; + + adaptor.addChild(root_0, bitwise_or11.getTree()); + + // src/java/org/apache/lucene/expressions/js/Javascript.g:263:18: ( AT_BOOL_AND ^ bitwise_or )* + loop3: + while (true) { + int alt3=2; + int LA3_0 = input.LA(1); + if ( (LA3_0==AT_BOOL_AND) ) { + alt3=1; + } + + switch (alt3) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:263:19: AT_BOOL_AND ^ bitwise_or + { + AT_BOOL_AND12=(Token)match(input,AT_BOOL_AND,FOLLOW_AT_BOOL_AND_in_logical_and815); + AT_BOOL_AND12_tree = (CommonTree)adaptor.create(AT_BOOL_AND12); + root_0 = (CommonTree)adaptor.becomeRoot(AT_BOOL_AND12_tree, root_0); + + pushFollow(FOLLOW_bitwise_or_in_logical_and818); + bitwise_or13=bitwise_or(); + state._fsp--; + + adaptor.addChild(root_0, bitwise_or13.getTree()); + + } + break; + + default : + break loop3; + } + } + + } + + retval.stop = input.LT(-1); + + retval.tree = (CommonTree)adaptor.rulePostProcessing(root_0); + adaptor.setTokenBoundaries(retval.tree, retval.start, retval.stop); + + } + catch (RecognitionException re) { + reportError(re); + recover(input,re); + retval.tree = (CommonTree)adaptor.errorNode(input, retval.start, input.LT(-1), re); + } + finally { + // do for sure before leaving + } + return retval; + } + // $ANTLR end "logical_and" + + + public static class bitwise_or_return extends ParserRuleReturnScope { + CommonTree tree; + @Override + public CommonTree getTree() { return tree; } + }; + + + // $ANTLR start "bitwise_or" + // src/java/org/apache/lucene/expressions/js/Javascript.g:266:1: bitwise_or : bitwise_xor ( AT_BIT_OR ^ bitwise_xor )* ; + public final XJavascriptParser.bitwise_or_return bitwise_or() throws RecognitionException { + XJavascriptParser.bitwise_or_return retval = new XJavascriptParser.bitwise_or_return(); + retval.start = input.LT(1); + + CommonTree root_0 = null; + + Token AT_BIT_OR15=null; + ParserRuleReturnScope bitwise_xor14 =null; + ParserRuleReturnScope bitwise_xor16 =null; + + CommonTree AT_BIT_OR15_tree=null; + + try { + // src/java/org/apache/lucene/expressions/js/Javascript.g:267:5: ( bitwise_xor ( AT_BIT_OR ^ bitwise_xor )* ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:267:7: bitwise_xor ( AT_BIT_OR ^ bitwise_xor )* + { + root_0 = (CommonTree)adaptor.nil(); + + + pushFollow(FOLLOW_bitwise_xor_in_bitwise_or837); + bitwise_xor14=bitwise_xor(); + state._fsp--; + + adaptor.addChild(root_0, bitwise_xor14.getTree()); + + // src/java/org/apache/lucene/expressions/js/Javascript.g:267:19: ( AT_BIT_OR ^ bitwise_xor )* + loop4: + while (true) { + int alt4=2; + int LA4_0 = input.LA(1); + if ( (LA4_0==AT_BIT_OR) ) { + alt4=1; + } + + switch (alt4) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:267:20: AT_BIT_OR ^ bitwise_xor + { + AT_BIT_OR15=(Token)match(input,AT_BIT_OR,FOLLOW_AT_BIT_OR_in_bitwise_or840); + AT_BIT_OR15_tree = (CommonTree)adaptor.create(AT_BIT_OR15); + root_0 = (CommonTree)adaptor.becomeRoot(AT_BIT_OR15_tree, root_0); + + pushFollow(FOLLOW_bitwise_xor_in_bitwise_or843); + bitwise_xor16=bitwise_xor(); + state._fsp--; + + adaptor.addChild(root_0, bitwise_xor16.getTree()); + + } + break; + + default : + break loop4; + } + } + + } + + retval.stop = input.LT(-1); + + retval.tree = (CommonTree)adaptor.rulePostProcessing(root_0); + adaptor.setTokenBoundaries(retval.tree, retval.start, retval.stop); + + } + catch (RecognitionException re) { + reportError(re); + recover(input,re); + retval.tree = (CommonTree)adaptor.errorNode(input, retval.start, input.LT(-1), re); + } + finally { + // do for sure before leaving + } + return retval; + } + // $ANTLR end "bitwise_or" + + + public static class bitwise_xor_return extends ParserRuleReturnScope { + CommonTree tree; + @Override + public CommonTree getTree() { return tree; } + }; + + + // $ANTLR start "bitwise_xor" + // src/java/org/apache/lucene/expressions/js/Javascript.g:270:1: bitwise_xor : bitwise_and ( AT_BIT_XOR ^ bitwise_and )* ; + public final XJavascriptParser.bitwise_xor_return bitwise_xor() throws RecognitionException { + XJavascriptParser.bitwise_xor_return retval = new XJavascriptParser.bitwise_xor_return(); + retval.start = input.LT(1); + + CommonTree root_0 = null; + + Token AT_BIT_XOR18=null; + ParserRuleReturnScope bitwise_and17 =null; + ParserRuleReturnScope bitwise_and19 =null; + + CommonTree AT_BIT_XOR18_tree=null; + + try { + // src/java/org/apache/lucene/expressions/js/Javascript.g:271:5: ( bitwise_and ( AT_BIT_XOR ^ bitwise_and )* ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:271:7: bitwise_and ( AT_BIT_XOR ^ bitwise_and )* + { + root_0 = (CommonTree)adaptor.nil(); + + + pushFollow(FOLLOW_bitwise_and_in_bitwise_xor862); + bitwise_and17=bitwise_and(); + state._fsp--; + + adaptor.addChild(root_0, bitwise_and17.getTree()); + + // src/java/org/apache/lucene/expressions/js/Javascript.g:271:19: ( AT_BIT_XOR ^ bitwise_and )* + loop5: + while (true) { + int alt5=2; + int LA5_0 = input.LA(1); + if ( (LA5_0==AT_BIT_XOR) ) { + alt5=1; + } + + switch (alt5) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:271:20: AT_BIT_XOR ^ bitwise_and + { + AT_BIT_XOR18=(Token)match(input,AT_BIT_XOR,FOLLOW_AT_BIT_XOR_in_bitwise_xor865); + AT_BIT_XOR18_tree = (CommonTree)adaptor.create(AT_BIT_XOR18); + root_0 = (CommonTree)adaptor.becomeRoot(AT_BIT_XOR18_tree, root_0); + + pushFollow(FOLLOW_bitwise_and_in_bitwise_xor868); + bitwise_and19=bitwise_and(); + state._fsp--; + + adaptor.addChild(root_0, bitwise_and19.getTree()); + + } + break; + + default : + break loop5; + } + } + + } + + retval.stop = input.LT(-1); + + retval.tree = (CommonTree)adaptor.rulePostProcessing(root_0); + adaptor.setTokenBoundaries(retval.tree, retval.start, retval.stop); + + } + catch (RecognitionException re) { + reportError(re); + recover(input,re); + retval.tree = (CommonTree)adaptor.errorNode(input, retval.start, input.LT(-1), re); + } + finally { + // do for sure before leaving + } + return retval; + } + // $ANTLR end "bitwise_xor" + + + public static class bitwise_and_return extends ParserRuleReturnScope { + CommonTree tree; + @Override + public CommonTree getTree() { return tree; } + }; + + + // $ANTLR start "bitwise_and" + // src/java/org/apache/lucene/expressions/js/Javascript.g:274:1: bitwise_and : equality ( AT_BIT_AND ^ equality )* ; + public final XJavascriptParser.bitwise_and_return bitwise_and() throws RecognitionException { + XJavascriptParser.bitwise_and_return retval = new XJavascriptParser.bitwise_and_return(); + retval.start = input.LT(1); + + CommonTree root_0 = null; + + Token AT_BIT_AND21=null; + ParserRuleReturnScope equality20 =null; + ParserRuleReturnScope equality22 =null; + + CommonTree AT_BIT_AND21_tree=null; + + try { + // src/java/org/apache/lucene/expressions/js/Javascript.g:275:5: ( equality ( AT_BIT_AND ^ equality )* ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:275:8: equality ( AT_BIT_AND ^ equality )* + { + root_0 = (CommonTree)adaptor.nil(); + + + pushFollow(FOLLOW_equality_in_bitwise_and888); + equality20=equality(); + state._fsp--; + + adaptor.addChild(root_0, equality20.getTree()); + + // src/java/org/apache/lucene/expressions/js/Javascript.g:275:17: ( AT_BIT_AND ^ equality )* + loop6: + while (true) { + int alt6=2; + int LA6_0 = input.LA(1); + if ( (LA6_0==AT_BIT_AND) ) { + alt6=1; + } + + switch (alt6) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:275:18: AT_BIT_AND ^ equality + { + AT_BIT_AND21=(Token)match(input,AT_BIT_AND,FOLLOW_AT_BIT_AND_in_bitwise_and891); + AT_BIT_AND21_tree = (CommonTree)adaptor.create(AT_BIT_AND21); + root_0 = (CommonTree)adaptor.becomeRoot(AT_BIT_AND21_tree, root_0); + + pushFollow(FOLLOW_equality_in_bitwise_and894); + equality22=equality(); + state._fsp--; + + adaptor.addChild(root_0, equality22.getTree()); + + } + break; + + default : + break loop6; + } + } + + } + + retval.stop = input.LT(-1); + + retval.tree = (CommonTree)adaptor.rulePostProcessing(root_0); + adaptor.setTokenBoundaries(retval.tree, retval.start, retval.stop); + + } + catch (RecognitionException re) { + reportError(re); + recover(input,re); + retval.tree = (CommonTree)adaptor.errorNode(input, retval.start, input.LT(-1), re); + } + finally { + // do for sure before leaving + } + return retval; + } + // $ANTLR end "bitwise_and" + + + public static class equality_return extends ParserRuleReturnScope { + CommonTree tree; + @Override + public CommonTree getTree() { return tree; } + }; + + + // $ANTLR start "equality" + // src/java/org/apache/lucene/expressions/js/Javascript.g:278:1: equality : relational ( ( AT_COMP_EQ | AT_COMP_NEQ ) ^ relational )* ; + public final XJavascriptParser.equality_return equality() throws RecognitionException { + XJavascriptParser.equality_return retval = new XJavascriptParser.equality_return(); + retval.start = input.LT(1); + + CommonTree root_0 = null; + + Token set24=null; + ParserRuleReturnScope relational23 =null; + ParserRuleReturnScope relational25 =null; + + CommonTree set24_tree=null; + + try { + // src/java/org/apache/lucene/expressions/js/Javascript.g:279:5: ( relational ( ( AT_COMP_EQ | AT_COMP_NEQ ) ^ relational )* ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:279:7: relational ( ( AT_COMP_EQ | AT_COMP_NEQ ) ^ relational )* + { + root_0 = (CommonTree)adaptor.nil(); + + + pushFollow(FOLLOW_relational_in_equality913); + relational23=relational(); + state._fsp--; + + adaptor.addChild(root_0, relational23.getTree()); + + // src/java/org/apache/lucene/expressions/js/Javascript.g:279:18: ( ( AT_COMP_EQ | AT_COMP_NEQ ) ^ relational )* + loop7: + while (true) { + int alt7=2; + int LA7_0 = input.LA(1); + if ( (LA7_0==AT_COMP_EQ||LA7_0==AT_COMP_NEQ) ) { + alt7=1; + } + + switch (alt7) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:279:19: ( AT_COMP_EQ | AT_COMP_NEQ ) ^ relational + { + set24=input.LT(1); + set24=input.LT(1); + if ( input.LA(1)==AT_COMP_EQ||input.LA(1)==AT_COMP_NEQ ) { + input.consume(); + root_0 = (CommonTree)adaptor.becomeRoot((CommonTree)adaptor.create(set24), root_0); + state.errorRecovery=false; + } + else { + MismatchedSetException mse = new MismatchedSetException(null,input); + throw mse; + } + pushFollow(FOLLOW_relational_in_equality925); + relational25=relational(); + state._fsp--; + + adaptor.addChild(root_0, relational25.getTree()); + + } + break; + + default : + break loop7; + } + } + + } + + retval.stop = input.LT(-1); + + retval.tree = (CommonTree)adaptor.rulePostProcessing(root_0); + adaptor.setTokenBoundaries(retval.tree, retval.start, retval.stop); + + } + catch (RecognitionException re) { + reportError(re); + recover(input,re); + retval.tree = (CommonTree)adaptor.errorNode(input, retval.start, input.LT(-1), re); + } + finally { + // do for sure before leaving + } + return retval; + } + // $ANTLR end "equality" + + + public static class relational_return extends ParserRuleReturnScope { + CommonTree tree; + @Override + public CommonTree getTree() { return tree; } + }; + + + // $ANTLR start "relational" + // src/java/org/apache/lucene/expressions/js/Javascript.g:282:1: relational : shift ( ( AT_COMP_LT | AT_COMP_GT | AT_COMP_LTE | AT_COMP_GTE ) ^ shift )* ; + public final XJavascriptParser.relational_return relational() throws RecognitionException { + XJavascriptParser.relational_return retval = new XJavascriptParser.relational_return(); + retval.start = input.LT(1); + + CommonTree root_0 = null; + + Token set27=null; + ParserRuleReturnScope shift26 =null; + ParserRuleReturnScope shift28 =null; + + CommonTree set27_tree=null; + + try { + // src/java/org/apache/lucene/expressions/js/Javascript.g:283:5: ( shift ( ( AT_COMP_LT | AT_COMP_GT | AT_COMP_LTE | AT_COMP_GTE ) ^ shift )* ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:283:7: shift ( ( AT_COMP_LT | AT_COMP_GT | AT_COMP_LTE | AT_COMP_GTE ) ^ shift )* + { + root_0 = (CommonTree)adaptor.nil(); + + + pushFollow(FOLLOW_shift_in_relational944); + shift26=shift(); + state._fsp--; + + adaptor.addChild(root_0, shift26.getTree()); + + // src/java/org/apache/lucene/expressions/js/Javascript.g:283:13: ( ( AT_COMP_LT | AT_COMP_GT | AT_COMP_LTE | AT_COMP_GTE ) ^ shift )* + loop8: + while (true) { + int alt8=2; + int LA8_0 = input.LA(1); + if ( ((LA8_0 >= AT_COMP_GT && LA8_0 <= AT_COMP_LTE)) ) { + alt8=1; + } + + switch (alt8) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:283:14: ( AT_COMP_LT | AT_COMP_GT | AT_COMP_LTE | AT_COMP_GTE ) ^ shift + { + set27=input.LT(1); + set27=input.LT(1); + if ( (input.LA(1) >= AT_COMP_GT && input.LA(1) <= AT_COMP_LTE) ) { + input.consume(); + root_0 = (CommonTree)adaptor.becomeRoot((CommonTree)adaptor.create(set27), root_0); + state.errorRecovery=false; + } + else { + MismatchedSetException mse = new MismatchedSetException(null,input); + throw mse; + } + pushFollow(FOLLOW_shift_in_relational964); + shift28=shift(); + state._fsp--; + + adaptor.addChild(root_0, shift28.getTree()); + + } + break; + + default : + break loop8; + } + } + + } + + retval.stop = input.LT(-1); + + retval.tree = (CommonTree)adaptor.rulePostProcessing(root_0); + adaptor.setTokenBoundaries(retval.tree, retval.start, retval.stop); + + } + catch (RecognitionException re) { + reportError(re); + recover(input,re); + retval.tree = (CommonTree)adaptor.errorNode(input, retval.start, input.LT(-1), re); + } + finally { + // do for sure before leaving + } + return retval; + } + // $ANTLR end "relational" + + + public static class shift_return extends ParserRuleReturnScope { + CommonTree tree; + @Override + public CommonTree getTree() { return tree; } + }; + + + // $ANTLR start "shift" + // src/java/org/apache/lucene/expressions/js/Javascript.g:286:1: shift : additive ( ( AT_BIT_SHL | AT_BIT_SHR | AT_BIT_SHU ) ^ additive )* ; + public final XJavascriptParser.shift_return shift() throws RecognitionException { + XJavascriptParser.shift_return retval = new XJavascriptParser.shift_return(); + retval.start = input.LT(1); + + CommonTree root_0 = null; + + Token set30=null; + ParserRuleReturnScope additive29 =null; + ParserRuleReturnScope additive31 =null; + + CommonTree set30_tree=null; + + try { + // src/java/org/apache/lucene/expressions/js/Javascript.g:287:5: ( additive ( ( AT_BIT_SHL | AT_BIT_SHR | AT_BIT_SHU ) ^ additive )* ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:287:7: additive ( ( AT_BIT_SHL | AT_BIT_SHR | AT_BIT_SHU ) ^ additive )* + { + root_0 = (CommonTree)adaptor.nil(); + + + pushFollow(FOLLOW_additive_in_shift983); + additive29=additive(); + state._fsp--; + + adaptor.addChild(root_0, additive29.getTree()); + + // src/java/org/apache/lucene/expressions/js/Javascript.g:287:16: ( ( AT_BIT_SHL | AT_BIT_SHR | AT_BIT_SHU ) ^ additive )* + loop9: + while (true) { + int alt9=2; + int LA9_0 = input.LA(1); + if ( ((LA9_0 >= AT_BIT_SHL && LA9_0 <= AT_BIT_SHU)) ) { + alt9=1; + } + + switch (alt9) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:287:17: ( AT_BIT_SHL | AT_BIT_SHR | AT_BIT_SHU ) ^ additive + { + set30=input.LT(1); + set30=input.LT(1); + if ( (input.LA(1) >= AT_BIT_SHL && input.LA(1) <= AT_BIT_SHU) ) { + input.consume(); + root_0 = (CommonTree)adaptor.becomeRoot((CommonTree)adaptor.create(set30), root_0); + state.errorRecovery=false; + } + else { + MismatchedSetException mse = new MismatchedSetException(null,input); + throw mse; + } + pushFollow(FOLLOW_additive_in_shift999); + additive31=additive(); + state._fsp--; + + adaptor.addChild(root_0, additive31.getTree()); + + } + break; + + default : + break loop9; + } + } + + } + + retval.stop = input.LT(-1); + + retval.tree = (CommonTree)adaptor.rulePostProcessing(root_0); + adaptor.setTokenBoundaries(retval.tree, retval.start, retval.stop); + + } + catch (RecognitionException re) { + reportError(re); + recover(input,re); + retval.tree = (CommonTree)adaptor.errorNode(input, retval.start, input.LT(-1), re); + } + finally { + // do for sure before leaving + } + return retval; + } + // $ANTLR end "shift" + + + public static class additive_return extends ParserRuleReturnScope { + CommonTree tree; + @Override + public CommonTree getTree() { return tree; } + }; + + + // $ANTLR start "additive" + // src/java/org/apache/lucene/expressions/js/Javascript.g:290:1: additive : multiplicative ( ( AT_ADD | AT_SUBTRACT ) ^ multiplicative )* ; + public final XJavascriptParser.additive_return additive() throws RecognitionException { + XJavascriptParser.additive_return retval = new XJavascriptParser.additive_return(); + retval.start = input.LT(1); + + CommonTree root_0 = null; + + Token set33=null; + ParserRuleReturnScope multiplicative32 =null; + ParserRuleReturnScope multiplicative34 =null; + + CommonTree set33_tree=null; + + try { + // src/java/org/apache/lucene/expressions/js/Javascript.g:291:5: ( multiplicative ( ( AT_ADD | AT_SUBTRACT ) ^ multiplicative )* ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:291:7: multiplicative ( ( AT_ADD | AT_SUBTRACT ) ^ multiplicative )* + { + root_0 = (CommonTree)adaptor.nil(); + + + pushFollow(FOLLOW_multiplicative_in_additive1018); + multiplicative32=multiplicative(); + state._fsp--; + + adaptor.addChild(root_0, multiplicative32.getTree()); + + // src/java/org/apache/lucene/expressions/js/Javascript.g:291:22: ( ( AT_ADD | AT_SUBTRACT ) ^ multiplicative )* + loop10: + while (true) { + int alt10=2; + int LA10_0 = input.LA(1); + if ( (LA10_0==AT_ADD||LA10_0==AT_SUBTRACT) ) { + alt10=1; + } + + switch (alt10) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:291:23: ( AT_ADD | AT_SUBTRACT ) ^ multiplicative + { + set33=input.LT(1); + set33=input.LT(1); + if ( input.LA(1)==AT_ADD||input.LA(1)==AT_SUBTRACT ) { + input.consume(); + root_0 = (CommonTree)adaptor.becomeRoot((CommonTree)adaptor.create(set33), root_0); + state.errorRecovery=false; + } + else { + MismatchedSetException mse = new MismatchedSetException(null,input); + throw mse; + } + pushFollow(FOLLOW_multiplicative_in_additive1030); + multiplicative34=multiplicative(); + state._fsp--; + + adaptor.addChild(root_0, multiplicative34.getTree()); + + } + break; + + default : + break loop10; + } + } + + } + + retval.stop = input.LT(-1); + + retval.tree = (CommonTree)adaptor.rulePostProcessing(root_0); + adaptor.setTokenBoundaries(retval.tree, retval.start, retval.stop); + + } + catch (RecognitionException re) { + reportError(re); + recover(input,re); + retval.tree = (CommonTree)adaptor.errorNode(input, retval.start, input.LT(-1), re); + } + finally { + // do for sure before leaving + } + return retval; + } + // $ANTLR end "additive" + + + public static class multiplicative_return extends ParserRuleReturnScope { + CommonTree tree; + @Override + public CommonTree getTree() { return tree; } + }; + + + // $ANTLR start "multiplicative" + // src/java/org/apache/lucene/expressions/js/Javascript.g:294:1: multiplicative : unary ( ( AT_MULTIPLY | AT_DIVIDE | AT_MODULO ) ^ unary )* ; + public final XJavascriptParser.multiplicative_return multiplicative() throws RecognitionException { + XJavascriptParser.multiplicative_return retval = new XJavascriptParser.multiplicative_return(); + retval.start = input.LT(1); + + CommonTree root_0 = null; + + Token set36=null; + ParserRuleReturnScope unary35 =null; + ParserRuleReturnScope unary37 =null; + + CommonTree set36_tree=null; + + try { + // src/java/org/apache/lucene/expressions/js/Javascript.g:295:5: ( unary ( ( AT_MULTIPLY | AT_DIVIDE | AT_MODULO ) ^ unary )* ) + // src/java/org/apache/lucene/expressions/js/Javascript.g:295:7: unary ( ( AT_MULTIPLY | AT_DIVIDE | AT_MODULO ) ^ unary )* + { + root_0 = (CommonTree)adaptor.nil(); + + + pushFollow(FOLLOW_unary_in_multiplicative1049); + unary35=unary(); + state._fsp--; + + adaptor.addChild(root_0, unary35.getTree()); + + // src/java/org/apache/lucene/expressions/js/Javascript.g:295:13: ( ( AT_MULTIPLY | AT_DIVIDE | AT_MODULO ) ^ unary )* + loop11: + while (true) { + int alt11=2; + int LA11_0 = input.LA(1); + if ( (LA11_0==AT_DIVIDE||(LA11_0 >= AT_MODULO && LA11_0 <= AT_MULTIPLY)) ) { + alt11=1; + } + + switch (alt11) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:295:14: ( AT_MULTIPLY | AT_DIVIDE | AT_MODULO ) ^ unary + { + set36=input.LT(1); + set36=input.LT(1); + if ( input.LA(1)==AT_DIVIDE||(input.LA(1) >= AT_MODULO && input.LA(1) <= AT_MULTIPLY) ) { + input.consume(); + root_0 = (CommonTree)adaptor.becomeRoot((CommonTree)adaptor.create(set36), root_0); + state.errorRecovery=false; + } + else { + MismatchedSetException mse = new MismatchedSetException(null,input); + throw mse; + } + pushFollow(FOLLOW_unary_in_multiplicative1065); + unary37=unary(); + state._fsp--; + + adaptor.addChild(root_0, unary37.getTree()); + + } + break; + + default : + break loop11; + } + } + + } + + retval.stop = input.LT(-1); + + retval.tree = (CommonTree)adaptor.rulePostProcessing(root_0); + adaptor.setTokenBoundaries(retval.tree, retval.start, retval.stop); + + } + catch (RecognitionException re) { + reportError(re); + recover(input,re); + retval.tree = (CommonTree)adaptor.errorNode(input, retval.start, input.LT(-1), re); + } + finally { + // do for sure before leaving + } + return retval; + } + // $ANTLR end "multiplicative" + + + public static class unary_return extends ParserRuleReturnScope { + CommonTree tree; + @Override + public CommonTree getTree() { return tree; } + }; + + + // $ANTLR start "unary" + // src/java/org/apache/lucene/expressions/js/Javascript.g:298:1: unary : ( postfix | AT_ADD ! unary | unary_operator ^ unary ); + public final XJavascriptParser.unary_return unary() throws RecognitionException { + XJavascriptParser.unary_return retval = new XJavascriptParser.unary_return(); + retval.start = input.LT(1); + + CommonTree root_0 = null; + + Token AT_ADD39=null; + ParserRuleReturnScope postfix38 =null; + ParserRuleReturnScope unary40 =null; + ParserRuleReturnScope unary_operator41 =null; + ParserRuleReturnScope unary42 =null; + + CommonTree AT_ADD39_tree=null; + + try { + // src/java/org/apache/lucene/expressions/js/Javascript.g:299:5: ( postfix | AT_ADD ! unary | unary_operator ^ unary ) + int alt12=3; + switch ( input.LA(1) ) { + case AT_LPAREN: + case DECIMAL: + case HEX: + case OCTAL: + case VARIABLE: + { + alt12=1; + } + break; + case AT_ADD: + { + alt12=2; + } + break; + case AT_BIT_NOT: + case AT_BOOL_NOT: + case AT_SUBTRACT: + { + alt12=3; + } + break; + default: + NoViableAltException nvae = + new NoViableAltException("", 12, 0, input); + throw nvae; + } + switch (alt12) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:299:7: postfix + { + root_0 = (CommonTree)adaptor.nil(); + + + pushFollow(FOLLOW_postfix_in_unary1084); + postfix38=postfix(); + state._fsp--; + + adaptor.addChild(root_0, postfix38.getTree()); + + } + break; + case 2 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:300:7: AT_ADD ! unary + { + root_0 = (CommonTree)adaptor.nil(); + + + AT_ADD39=(Token)match(input,AT_ADD,FOLLOW_AT_ADD_in_unary1092); + pushFollow(FOLLOW_unary_in_unary1095); + unary40=unary(); + state._fsp--; + + adaptor.addChild(root_0, unary40.getTree()); + + } + break; + case 3 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:301:7: unary_operator ^ unary + { + root_0 = (CommonTree)adaptor.nil(); + + + pushFollow(FOLLOW_unary_operator_in_unary1103); + unary_operator41=unary_operator(); + state._fsp--; + + root_0 = (CommonTree)adaptor.becomeRoot(unary_operator41.getTree(), root_0); + pushFollow(FOLLOW_unary_in_unary1106); + unary42=unary(); + state._fsp--; + + adaptor.addChild(root_0, unary42.getTree()); + + } + break; + + } + retval.stop = input.LT(-1); + + retval.tree = (CommonTree)adaptor.rulePostProcessing(root_0); + adaptor.setTokenBoundaries(retval.tree, retval.start, retval.stop); + + } + catch (RecognitionException re) { + reportError(re); + recover(input,re); + retval.tree = (CommonTree)adaptor.errorNode(input, retval.start, input.LT(-1), re); + } + finally { + // do for sure before leaving + } + return retval; + } + // $ANTLR end "unary" + + + public static class unary_operator_return extends ParserRuleReturnScope { + CommonTree tree; + @Override + public CommonTree getTree() { return tree; } + }; + + + // $ANTLR start "unary_operator" + // src/java/org/apache/lucene/expressions/js/Javascript.g:304:1: unary_operator : ( AT_SUBTRACT -> AT_NEGATE | AT_BIT_NOT | AT_BOOL_NOT ); + public final XJavascriptParser.unary_operator_return unary_operator() throws RecognitionException { + XJavascriptParser.unary_operator_return retval = new XJavascriptParser.unary_operator_return(); + retval.start = input.LT(1); + + CommonTree root_0 = null; + + Token AT_SUBTRACT43=null; + Token AT_BIT_NOT44=null; + Token AT_BOOL_NOT45=null; + + CommonTree AT_SUBTRACT43_tree=null; + CommonTree AT_BIT_NOT44_tree=null; + CommonTree AT_BOOL_NOT45_tree=null; + RewriteRuleTokenStream stream_AT_SUBTRACT=new RewriteRuleTokenStream(adaptor,"token AT_SUBTRACT"); + + try { + // src/java/org/apache/lucene/expressions/js/Javascript.g:305:5: ( AT_SUBTRACT -> AT_NEGATE | AT_BIT_NOT | AT_BOOL_NOT ) + int alt13=3; + switch ( input.LA(1) ) { + case AT_SUBTRACT: + { + alt13=1; + } + break; + case AT_BIT_NOT: + { + alt13=2; + } + break; + case AT_BOOL_NOT: + { + alt13=3; + } + break; + default: + NoViableAltException nvae = + new NoViableAltException("", 13, 0, input); + throw nvae; + } + switch (alt13) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:305:7: AT_SUBTRACT + { + AT_SUBTRACT43=(Token)match(input,AT_SUBTRACT,FOLLOW_AT_SUBTRACT_in_unary_operator1123); + stream_AT_SUBTRACT.add(AT_SUBTRACT43); + + // AST REWRITE + // elements: + // token labels: + // rule labels: retval + // token list labels: + // rule list labels: + // wildcard labels: + retval.tree = root_0; + RewriteRuleSubtreeStream stream_retval=new RewriteRuleSubtreeStream(adaptor,"rule retval",retval!=null?retval.getTree():null); + + root_0 = (CommonTree)adaptor.nil(); + // 305:19: -> AT_NEGATE + { + adaptor.addChild(root_0, (CommonTree)adaptor.create(AT_NEGATE, "AT_NEGATE")); + } + + + retval.tree = root_0; + + } + break; + case 2 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:306:7: AT_BIT_NOT + { + root_0 = (CommonTree)adaptor.nil(); + + + AT_BIT_NOT44=(Token)match(input,AT_BIT_NOT,FOLLOW_AT_BIT_NOT_in_unary_operator1135); + AT_BIT_NOT44_tree = (CommonTree)adaptor.create(AT_BIT_NOT44); + adaptor.addChild(root_0, AT_BIT_NOT44_tree); + + } + break; + case 3 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:307:7: AT_BOOL_NOT + { + root_0 = (CommonTree)adaptor.nil(); + + + AT_BOOL_NOT45=(Token)match(input,AT_BOOL_NOT,FOLLOW_AT_BOOL_NOT_in_unary_operator1143); + AT_BOOL_NOT45_tree = (CommonTree)adaptor.create(AT_BOOL_NOT45); + adaptor.addChild(root_0, AT_BOOL_NOT45_tree); + + } + break; + + } + retval.stop = input.LT(-1); + + retval.tree = (CommonTree)adaptor.rulePostProcessing(root_0); + adaptor.setTokenBoundaries(retval.tree, retval.start, retval.stop); + + } + catch (RecognitionException re) { + reportError(re); + recover(input,re); + retval.tree = (CommonTree)adaptor.errorNode(input, retval.start, input.LT(-1), re); + } + finally { + // do for sure before leaving + } + return retval; + } + // $ANTLR end "unary_operator" + + + public static class postfix_return extends ParserRuleReturnScope { + CommonTree tree; + @Override + public CommonTree getTree() { return tree; } + }; + + + // $ANTLR start "postfix" + // src/java/org/apache/lucene/expressions/js/Javascript.g:310:1: postfix : ( primary | VARIABLE arguments -> ^( AT_CALL VARIABLE ( arguments )? ) ); + public final XJavascriptParser.postfix_return postfix() throws RecognitionException { + XJavascriptParser.postfix_return retval = new XJavascriptParser.postfix_return(); + retval.start = input.LT(1); + + CommonTree root_0 = null; + + Token VARIABLE47=null; + ParserRuleReturnScope primary46 =null; + ParserRuleReturnScope arguments48 =null; + + CommonTree VARIABLE47_tree=null; + RewriteRuleTokenStream stream_VARIABLE=new RewriteRuleTokenStream(adaptor,"token VARIABLE"); + RewriteRuleSubtreeStream stream_arguments=new RewriteRuleSubtreeStream(adaptor,"rule arguments"); + + try { + // src/java/org/apache/lucene/expressions/js/Javascript.g:311:5: ( primary | VARIABLE arguments -> ^( AT_CALL VARIABLE ( arguments )? ) ) + int alt14=2; + int LA14_0 = input.LA(1); + if ( (LA14_0==VARIABLE) ) { + int LA14_1 = input.LA(2); + if ( (LA14_1==EOF||(LA14_1 >= AT_ADD && LA14_1 <= AT_BIT_AND)||(LA14_1 >= AT_BIT_OR && LA14_1 <= AT_BOOL_AND)||LA14_1==AT_BOOL_OR||(LA14_1 >= AT_COLON && LA14_1 <= AT_DIVIDE)||(LA14_1 >= AT_MODULO && LA14_1 <= AT_MULTIPLY)||(LA14_1 >= AT_RPAREN && LA14_1 <= AT_SUBTRACT)) ) { + alt14=1; + } + else if ( (LA14_1==AT_LPAREN) ) { + alt14=2; + } + + else { + int nvaeMark = input.mark(); + try { + input.consume(); + NoViableAltException nvae = + new NoViableAltException("", 14, 1, input); + throw nvae; + } finally { + input.rewind(nvaeMark); + } + } + + } + else if ( (LA14_0==AT_LPAREN||LA14_0==DECIMAL||LA14_0==HEX||LA14_0==OCTAL) ) { + alt14=1; + } + + else { + NoViableAltException nvae = + new NoViableAltException("", 14, 0, input); + throw nvae; + } + + switch (alt14) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:311:7: primary + { + root_0 = (CommonTree)adaptor.nil(); + + + pushFollow(FOLLOW_primary_in_postfix1160); + primary46=primary(); + state._fsp--; + + adaptor.addChild(root_0, primary46.getTree()); + + } + break; + case 2 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:312:7: VARIABLE arguments + { + VARIABLE47=(Token)match(input,VARIABLE,FOLLOW_VARIABLE_in_postfix1168); + stream_VARIABLE.add(VARIABLE47); + + pushFollow(FOLLOW_arguments_in_postfix1170); + arguments48=arguments(); + state._fsp--; + + stream_arguments.add(arguments48.getTree()); + // AST REWRITE + // elements: VARIABLE, arguments + // token labels: + // rule labels: retval + // token list labels: + // rule list labels: + // wildcard labels: + retval.tree = root_0; + RewriteRuleSubtreeStream stream_retval=new RewriteRuleSubtreeStream(adaptor,"rule retval",retval!=null?retval.getTree():null); + + root_0 = (CommonTree)adaptor.nil(); + // 312:26: -> ^( AT_CALL VARIABLE ( arguments )? ) + { + // src/java/org/apache/lucene/expressions/js/Javascript.g:312:29: ^( AT_CALL VARIABLE ( arguments )? ) + { + CommonTree root_1 = (CommonTree)adaptor.nil(); + root_1 = (CommonTree)adaptor.becomeRoot((CommonTree)adaptor.create(AT_CALL, "AT_CALL"), root_1); + adaptor.addChild(root_1, stream_VARIABLE.nextNode()); + // src/java/org/apache/lucene/expressions/js/Javascript.g:312:48: ( arguments )? + if ( stream_arguments.hasNext() ) { + adaptor.addChild(root_1, stream_arguments.nextTree()); + } + stream_arguments.reset(); + + adaptor.addChild(root_0, root_1); + } + + } + + + retval.tree = root_0; + + } + break; + + } + retval.stop = input.LT(-1); + + retval.tree = (CommonTree)adaptor.rulePostProcessing(root_0); + adaptor.setTokenBoundaries(retval.tree, retval.start, retval.stop); + + } + catch (RecognitionException re) { + reportError(re); + recover(input,re); + retval.tree = (CommonTree)adaptor.errorNode(input, retval.start, input.LT(-1), re); + } + finally { + // do for sure before leaving + } + return retval; + } + // $ANTLR end "postfix" + + + public static class primary_return extends ParserRuleReturnScope { + CommonTree tree; + @Override + public CommonTree getTree() { return tree; } + }; + + + // $ANTLR start "primary" + // src/java/org/apache/lucene/expressions/js/Javascript.g:315:1: primary : ( VARIABLE | numeric | AT_LPAREN ! conditional AT_RPAREN !); + public final XJavascriptParser.primary_return primary() throws RecognitionException { + XJavascriptParser.primary_return retval = new XJavascriptParser.primary_return(); + retval.start = input.LT(1); + + CommonTree root_0 = null; + + Token VARIABLE49=null; + Token AT_LPAREN51=null; + Token AT_RPAREN53=null; + ParserRuleReturnScope numeric50 =null; + ParserRuleReturnScope conditional52 =null; + + CommonTree VARIABLE49_tree=null; + CommonTree AT_LPAREN51_tree=null; + CommonTree AT_RPAREN53_tree=null; + + try { + // src/java/org/apache/lucene/expressions/js/Javascript.g:316:5: ( VARIABLE | numeric | AT_LPAREN ! conditional AT_RPAREN !) + int alt15=3; + switch ( input.LA(1) ) { + case VARIABLE: + { + alt15=1; + } + break; + case DECIMAL: + case HEX: + case OCTAL: + { + alt15=2; + } + break; + case AT_LPAREN: + { + alt15=3; + } + break; + default: + NoViableAltException nvae = + new NoViableAltException("", 15, 0, input); + throw nvae; + } + switch (alt15) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:316:7: VARIABLE + { + root_0 = (CommonTree)adaptor.nil(); + + + VARIABLE49=(Token)match(input,VARIABLE,FOLLOW_VARIABLE_in_primary1198); + VARIABLE49_tree = (CommonTree)adaptor.create(VARIABLE49); + adaptor.addChild(root_0, VARIABLE49_tree); + + } + break; + case 2 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:317:7: numeric + { + root_0 = (CommonTree)adaptor.nil(); + + + pushFollow(FOLLOW_numeric_in_primary1206); + numeric50=numeric(); + state._fsp--; + + adaptor.addChild(root_0, numeric50.getTree()); + + } + break; + case 3 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:318:7: AT_LPAREN ! conditional AT_RPAREN ! + { + root_0 = (CommonTree)adaptor.nil(); + + + AT_LPAREN51=(Token)match(input,AT_LPAREN,FOLLOW_AT_LPAREN_in_primary1214); + pushFollow(FOLLOW_conditional_in_primary1217); + conditional52=conditional(); + state._fsp--; + + adaptor.addChild(root_0, conditional52.getTree()); + + AT_RPAREN53=(Token)match(input,AT_RPAREN,FOLLOW_AT_RPAREN_in_primary1219); + } + break; + + } + retval.stop = input.LT(-1); + + retval.tree = (CommonTree)adaptor.rulePostProcessing(root_0); + adaptor.setTokenBoundaries(retval.tree, retval.start, retval.stop); + + } + catch (RecognitionException re) { + reportError(re); + recover(input,re); + retval.tree = (CommonTree)adaptor.errorNode(input, retval.start, input.LT(-1), re); + } + finally { + // do for sure before leaving + } + return retval; + } + // $ANTLR end "primary" + + + public static class arguments_return extends ParserRuleReturnScope { + CommonTree tree; + @Override + public CommonTree getTree() { return tree; } + }; + + + // $ANTLR start "arguments" + // src/java/org/apache/lucene/expressions/js/Javascript.g:321:1: arguments : AT_LPAREN ! ( conditional ( AT_COMMA ! conditional )* )? AT_RPAREN !; + public final XJavascriptParser.arguments_return arguments() throws RecognitionException { + XJavascriptParser.arguments_return retval = new XJavascriptParser.arguments_return(); + retval.start = input.LT(1); + + CommonTree root_0 = null; + + Token AT_LPAREN54=null; + Token AT_COMMA56=null; + Token AT_RPAREN58=null; + ParserRuleReturnScope conditional55 =null; + ParserRuleReturnScope conditional57 =null; + + CommonTree AT_LPAREN54_tree=null; + CommonTree AT_COMMA56_tree=null; + CommonTree AT_RPAREN58_tree=null; + + try { + // src/java/org/apache/lucene/expressions/js/Javascript.g:322:5: ( AT_LPAREN ! ( conditional ( AT_COMMA ! conditional )* )? AT_RPAREN !) + // src/java/org/apache/lucene/expressions/js/Javascript.g:322:7: AT_LPAREN ! ( conditional ( AT_COMMA ! conditional )* )? AT_RPAREN ! + { + root_0 = (CommonTree)adaptor.nil(); + + + AT_LPAREN54=(Token)match(input,AT_LPAREN,FOLLOW_AT_LPAREN_in_arguments1237); + // src/java/org/apache/lucene/expressions/js/Javascript.g:322:18: ( conditional ( AT_COMMA ! conditional )* )? + int alt17=2; + int LA17_0 = input.LA(1); + if ( (LA17_0==AT_ADD||LA17_0==AT_BIT_NOT||LA17_0==AT_BOOL_NOT||LA17_0==AT_LPAREN||(LA17_0 >= AT_SUBTRACT && LA17_0 <= DECIMAL)||LA17_0==HEX||LA17_0==OCTAL||LA17_0==VARIABLE) ) { + alt17=1; + } + switch (alt17) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:322:19: conditional ( AT_COMMA ! conditional )* + { + pushFollow(FOLLOW_conditional_in_arguments1241); + conditional55=conditional(); + state._fsp--; + + adaptor.addChild(root_0, conditional55.getTree()); + + // src/java/org/apache/lucene/expressions/js/Javascript.g:322:31: ( AT_COMMA ! conditional )* + loop16: + while (true) { + int alt16=2; + int LA16_0 = input.LA(1); + if ( (LA16_0==AT_COMMA) ) { + alt16=1; + } + + switch (alt16) { + case 1 : + // src/java/org/apache/lucene/expressions/js/Javascript.g:322:32: AT_COMMA ! conditional + { + AT_COMMA56=(Token)match(input,AT_COMMA,FOLLOW_AT_COMMA_in_arguments1244); + pushFollow(FOLLOW_conditional_in_arguments1247); + conditional57=conditional(); + state._fsp--; + + adaptor.addChild(root_0, conditional57.getTree()); + + } + break; + + default : + break loop16; + } + } + + } + break; + + } + + AT_RPAREN58=(Token)match(input,AT_RPAREN,FOLLOW_AT_RPAREN_in_arguments1253); + } + + retval.stop = input.LT(-1); + + retval.tree = (CommonTree)adaptor.rulePostProcessing(root_0); + adaptor.setTokenBoundaries(retval.tree, retval.start, retval.stop); + + } + catch (RecognitionException re) { + reportError(re); + recover(input,re); + retval.tree = (CommonTree)adaptor.errorNode(input, retval.start, input.LT(-1), re); + } + finally { + // do for sure before leaving + } + return retval; + } + // $ANTLR end "arguments" + + + public static class numeric_return extends ParserRuleReturnScope { + CommonTree tree; + @Override + public CommonTree getTree() { return tree; } + }; + + + // $ANTLR start "numeric" + // src/java/org/apache/lucene/expressions/js/Javascript.g:325:1: numeric : ( HEX | OCTAL | DECIMAL ); + public final XJavascriptParser.numeric_return numeric() throws RecognitionException { + XJavascriptParser.numeric_return retval = new XJavascriptParser.numeric_return(); + retval.start = input.LT(1); + + CommonTree root_0 = null; + + Token set59=null; + + CommonTree set59_tree=null; + + try { + // src/java/org/apache/lucene/expressions/js/Javascript.g:326:5: ( HEX | OCTAL | DECIMAL ) + // src/java/org/apache/lucene/expressions/js/Javascript.g: + { + root_0 = (CommonTree)adaptor.nil(); + + + set59=input.LT(1); + if ( input.LA(1)==DECIMAL||input.LA(1)==HEX||input.LA(1)==OCTAL ) { + input.consume(); + adaptor.addChild(root_0, (CommonTree)adaptor.create(set59)); + state.errorRecovery=false; + } + else { + MismatchedSetException mse = new MismatchedSetException(null,input); + throw mse; + } + } + + retval.stop = input.LT(-1); + + retval.tree = (CommonTree)adaptor.rulePostProcessing(root_0); + adaptor.setTokenBoundaries(retval.tree, retval.start, retval.stop); + + } + catch (RecognitionException re) { + reportError(re); + recover(input,re); + retval.tree = (CommonTree)adaptor.errorNode(input, retval.start, input.LT(-1), re); + } + finally { + // do for sure before leaving + } + return retval; + } + // $ANTLR end "numeric" + + // Delegated rules + + + + public static final BitSet FOLLOW_conditional_in_expression737 = new BitSet(new long[]{0x0000000000000000L}); + public static final BitSet FOLLOW_EOF_in_expression739 = new BitSet(new long[]{0x0000000000000002L}); + public static final BitSet FOLLOW_logical_or_in_conditional757 = new BitSet(new long[]{0x0000000002000002L}); + public static final BitSet FOLLOW_AT_COND_QUE_in_conditional760 = new BitSet(new long[]{0x00008886100040A0L}); + public static final BitSet FOLLOW_conditional_in_conditional763 = new BitSet(new long[]{0x0000000000020000L}); + public static final BitSet FOLLOW_AT_COLON_in_conditional765 = new BitSet(new long[]{0x00008886100040A0L}); + public static final BitSet FOLLOW_conditional_in_conditional768 = new BitSet(new long[]{0x0000000000000002L}); + public static final BitSet FOLLOW_logical_and_in_logical_or787 = new BitSet(new long[]{0x0000000000008002L}); + public static final BitSet FOLLOW_AT_BOOL_OR_in_logical_or790 = new BitSet(new long[]{0x00008886100040A0L}); + public static final BitSet FOLLOW_logical_and_in_logical_or793 = new BitSet(new long[]{0x0000000000008002L}); + public static final BitSet FOLLOW_bitwise_or_in_logical_and812 = new BitSet(new long[]{0x0000000000002002L}); + public static final BitSet FOLLOW_AT_BOOL_AND_in_logical_and815 = new BitSet(new long[]{0x00008886100040A0L}); + public static final BitSet FOLLOW_bitwise_or_in_logical_and818 = new BitSet(new long[]{0x0000000000002002L}); + public static final BitSet FOLLOW_bitwise_xor_in_bitwise_or837 = new BitSet(new long[]{0x0000000000000102L}); + public static final BitSet FOLLOW_AT_BIT_OR_in_bitwise_or840 = new BitSet(new long[]{0x00008886100040A0L}); + public static final BitSet FOLLOW_bitwise_xor_in_bitwise_or843 = new BitSet(new long[]{0x0000000000000102L}); + public static final BitSet FOLLOW_bitwise_and_in_bitwise_xor862 = new BitSet(new long[]{0x0000000000001002L}); + public static final BitSet FOLLOW_AT_BIT_XOR_in_bitwise_xor865 = new BitSet(new long[]{0x00008886100040A0L}); + public static final BitSet FOLLOW_bitwise_and_in_bitwise_xor868 = new BitSet(new long[]{0x0000000000001002L}); + public static final BitSet FOLLOW_equality_in_bitwise_and888 = new BitSet(new long[]{0x0000000000000042L}); + public static final BitSet FOLLOW_AT_BIT_AND_in_bitwise_and891 = new BitSet(new long[]{0x00008886100040A0L}); + public static final BitSet FOLLOW_equality_in_bitwise_and894 = new BitSet(new long[]{0x0000000000000042L}); + public static final BitSet FOLLOW_relational_in_equality913 = new BitSet(new long[]{0x0000000001080002L}); + public static final BitSet FOLLOW_set_in_equality916 = new BitSet(new long[]{0x00008886100040A0L}); + public static final BitSet FOLLOW_relational_in_equality925 = new BitSet(new long[]{0x0000000001080002L}); + public static final BitSet FOLLOW_shift_in_relational944 = new BitSet(new long[]{0x0000000000F00002L}); + public static final BitSet FOLLOW_set_in_relational947 = new BitSet(new long[]{0x00008886100040A0L}); + public static final BitSet FOLLOW_shift_in_relational964 = new BitSet(new long[]{0x0000000000F00002L}); + public static final BitSet FOLLOW_additive_in_shift983 = new BitSet(new long[]{0x0000000000000E02L}); + public static final BitSet FOLLOW_set_in_shift986 = new BitSet(new long[]{0x00008886100040A0L}); + public static final BitSet FOLLOW_additive_in_shift999 = new BitSet(new long[]{0x0000000000000E02L}); + public static final BitSet FOLLOW_multiplicative_in_additive1018 = new BitSet(new long[]{0x0000000200000022L}); + public static final BitSet FOLLOW_set_in_additive1021 = new BitSet(new long[]{0x00008886100040A0L}); + public static final BitSet FOLLOW_multiplicative_in_additive1030 = new BitSet(new long[]{0x0000000200000022L}); + public static final BitSet FOLLOW_unary_in_multiplicative1049 = new BitSet(new long[]{0x0000000064000002L}); + public static final BitSet FOLLOW_set_in_multiplicative1052 = new BitSet(new long[]{0x00008886100040A0L}); + public static final BitSet FOLLOW_unary_in_multiplicative1065 = new BitSet(new long[]{0x0000000064000002L}); + public static final BitSet FOLLOW_postfix_in_unary1084 = new BitSet(new long[]{0x0000000000000002L}); + public static final BitSet FOLLOW_AT_ADD_in_unary1092 = new BitSet(new long[]{0x00008886100040A0L}); + public static final BitSet FOLLOW_unary_in_unary1095 = new BitSet(new long[]{0x0000000000000002L}); + public static final BitSet FOLLOW_unary_operator_in_unary1103 = new BitSet(new long[]{0x00008886100040A0L}); + public static final BitSet FOLLOW_unary_in_unary1106 = new BitSet(new long[]{0x0000000000000002L}); + public static final BitSet FOLLOW_AT_SUBTRACT_in_unary_operator1123 = new BitSet(new long[]{0x0000000000000002L}); + public static final BitSet FOLLOW_AT_BIT_NOT_in_unary_operator1135 = new BitSet(new long[]{0x0000000000000002L}); + public static final BitSet FOLLOW_AT_BOOL_NOT_in_unary_operator1143 = new BitSet(new long[]{0x0000000000000002L}); + public static final BitSet FOLLOW_primary_in_postfix1160 = new BitSet(new long[]{0x0000000000000002L}); + public static final BitSet FOLLOW_VARIABLE_in_postfix1168 = new BitSet(new long[]{0x0000000010000000L}); + public static final BitSet FOLLOW_arguments_in_postfix1170 = new BitSet(new long[]{0x0000000000000002L}); + public static final BitSet FOLLOW_VARIABLE_in_primary1198 = new BitSet(new long[]{0x0000000000000002L}); + public static final BitSet FOLLOW_numeric_in_primary1206 = new BitSet(new long[]{0x0000000000000002L}); + public static final BitSet FOLLOW_AT_LPAREN_in_primary1214 = new BitSet(new long[]{0x00008886100040A0L}); + public static final BitSet FOLLOW_conditional_in_primary1217 = new BitSet(new long[]{0x0000000100000000L}); + public static final BitSet FOLLOW_AT_RPAREN_in_primary1219 = new BitSet(new long[]{0x0000000000000002L}); + public static final BitSet FOLLOW_AT_LPAREN_in_arguments1237 = new BitSet(new long[]{0x00008887100040A0L}); + public static final BitSet FOLLOW_conditional_in_arguments1241 = new BitSet(new long[]{0x0000000100040000L}); + public static final BitSet FOLLOW_AT_COMMA_in_arguments1244 = new BitSet(new long[]{0x00008886100040A0L}); + public static final BitSet FOLLOW_conditional_in_arguments1247 = new BitSet(new long[]{0x0000000100040000L}); + public static final BitSet FOLLOW_AT_RPAREN_in_arguments1253 = new BitSet(new long[]{0x0000000000000002L}); +} diff --git a/src/main/java/org/apache/lucene/expressions/js/XVariableContext.java b/src/main/java/org/apache/lucene/expressions/js/XVariableContext.java new file mode 100644 index 0000000000000..fa0382a1e0995 --- /dev/null +++ b/src/main/java/org/apache/lucene/expressions/js/XVariableContext.java @@ -0,0 +1,101 @@ +package org.apache.lucene.expressions.js; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.ArrayList; +import java.util.List; + +/** + * A helper to parse the context of a variable name, which is the base variable, followed by the + * sequence of array (integer or string indexed) and member accesses. + */ +public class XVariableContext { + public static enum Type { + MEMBER, // "dot" access + STR_INDEX, // brackets with a string + INT_INDEX // brackets with a positive integer + } + + public final Type type; + public final String text; + public final int integer; + + private XVariableContext(Type c, String s, int i) { + type = c; + text = s; + integer = i; + } + + /** + * Parses a normalized javascript variable. All strings in the variable should be single quoted, + * and no spaces (except possibly within strings). + */ + public static final XVariableContext[] parse(String variable) { + char[] text = variable.toCharArray(); + List contexts = new ArrayList<>(); + int i = addMember(text, 0, contexts); // base variable is a "member" of the global namespace + while (i < text.length) { + if (text[i] == '[') { + if (text[++i] == '\'') { + i = addStringIndex(text, i, contexts); + } else { + i = addIntIndex(text, i, contexts); + } + ++i; // move past end bracket + } else { // text[i] == '.', ie object member + i = addMember(text, i + 1, contexts); + } + } + return contexts.toArray(new XVariableContext[contexts.size()]); + } + + // i points to start of member name + private static int addMember(final char[] text, int i, List contexts) { + int j = i + 1; + while (j < text.length && text[j] != '[' && text[j] != '.') ++j; // find first array or member access + contexts.add(new XVariableContext(Type.MEMBER, new String(text, i, j - i), -1)); + return j; + } + + // i points to start of single quoted index + private static int addStringIndex(final char[] text, int i, List contexts) { + ++i; // move past quote + int j = i; + while (text[j] != '\'') { // find end of single quoted string + if (text[j] == '\\') ++j; // skip over escapes + ++j; + } + StringBuffer buf = new StringBuffer(j - i); // space for string, without end quote + while (i < j) { // copy string to buffer (without begin/end quotes) + if (text[i] == '\\') ++i; // unescape escapes + buf.append(text[i]); + ++i; + } + contexts.add(new XVariableContext(Type.STR_INDEX, buf.toString(), -1)); + return j + 1; // move past quote, return end bracket location + } + + // i points to start of integer index + private static int addIntIndex(final char[] text, int i, List contexts) { + int j = i + 1; + while (text[j] != ']') ++j; // find end of array access + int index = Integer.parseInt(new String(text, i, j - i)); + contexts.add(new XVariableContext(Type.INT_INDEX, null, index)); + return j ; + } +} diff --git a/src/main/java/org/elasticsearch/script/expression/ExpressionScript.java b/src/main/java/org/elasticsearch/script/expression/ExpressionScript.java index fa3f7b6d898ce..559430c58eb53 100644 --- a/src/main/java/org/elasticsearch/script/expression/ExpressionScript.java +++ b/src/main/java/org/elasticsearch/script/expression/ExpressionScript.java @@ -1,3 +1,22 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.elasticsearch.script.expression; import org.apache.lucene.expressions.Bindings; @@ -19,7 +38,7 @@ class ExpressionScript implements SearchScript { /** Fake scorer for a single document */ - class CannedScorer extends Scorer { + static class CannedScorer extends Scorer { protected int docid; protected float score; @@ -117,7 +136,7 @@ public void setNextSource(Map source) { @Override public void setNextVar(String name, Object value) { - // nocommit: comment on why this isn't needed...wtf is it? + // nono commit: comment on why this isn't needed...wtf is it? } diff --git a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptBindings.java b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptBindings.java deleted file mode 100644 index 0d2e8ee4c9ae9..0000000000000 --- a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptBindings.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.elasticsearch.script.expression; - -import org.apache.lucene.expressions.Bindings; -import org.apache.lucene.queries.function.ValueSource; -import org.apache.lucene.queries.function.valuesource.DoubleConstValueSource; -import org.elasticsearch.index.fielddata.IndexFieldData; - -import java.util.HashMap; -import java.util.Map; - -/** - * TODO: We could get rid of this entirely if SimpleBindings had add(String, ValueSource) instead of only add(SortField) - */ -class ExpressionScriptBindings extends Bindings { - - Map variables = new HashMap<>(); - - void addConstant(String variable, double value) { - variables.put(variable, new DoubleConstValueSource(value)); - } - - void addField(String variable, IndexFieldData fieldData) { - variables.put(variable, new ExpressionScriptValueSource(fieldData)); - } - - @Override - public ValueSource getValueSource(String variable) { - // TODO: is _score a constant anywhere? - if (variable.equals("_score")) { - return getScoreValueSource(); - } else { - return variables.get(variable); - } - } -} diff --git a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptCompilationException.java b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptCompilationException.java index cd56cc410ccd3..48a3df1be65f6 100644 --- a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptCompilationException.java +++ b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptCompilationException.java @@ -1,3 +1,22 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.elasticsearch.script.expression; import org.elasticsearch.ElasticsearchException; diff --git a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java index 151df47bd9168..2b40e44432b68 100644 --- a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java +++ b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java @@ -1,14 +1,34 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.elasticsearch.script.expression; import org.apache.lucene.expressions.Expression; -import org.apache.lucene.expressions.SimpleBindings; -import org.apache.lucene.index.AtomicReaderContext; -import org.elasticsearch.ElasticsearchException; +import org.apache.lucene.expressions.XSimpleBindings; +import org.apache.lucene.expressions.js.XJavascriptCompiler; +import org.apache.lucene.expressions.js.XVariableContext; +import org.apache.lucene.queries.function.valuesource.DoubleConstValueSource; +import org.apache.lucene.search.SortField; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.fielddata.AtomicNumericFieldData; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.MapperService; @@ -18,11 +38,8 @@ import org.elasticsearch.script.SearchScript; import org.elasticsearch.search.lookup.SearchLookup; -import org.apache.lucene.expressions.js.JavascriptCompiler; - import java.text.ParseException; import java.util.Map; -import java.util.concurrent.atomic.AtomicLong; /** * Provides the infrastructure for Lucene expressions as a scripting language for Elasticsearch. Only @@ -30,13 +47,9 @@ */ public class ExpressionScriptEngineService extends AbstractComponent implements ScriptEngineService { - private final AtomicLong counter = new AtomicLong(); - private final ClassLoader classLoader; // TODO: should use this instead of the implicit this.getClass().getClassLoader()? - @Inject public ExpressionScriptEngineService(Settings settings) { super(settings); - classLoader = settings.getClassLoader(); } @Override @@ -58,7 +71,7 @@ public boolean sandboxed() { public Object compile(String script) { try { // NOTE: validation is delayed to allow runtime vars, and we don't have access to per index stuff here - return JavascriptCompiler.compile(script); + return XJavascriptCompiler.compile(script); } catch (ParseException e) { throw new ExpressionScriptCompilationException("Failed to parse expression: " + script, e); } @@ -68,34 +81,47 @@ public Object compile(String script) { public SearchScript search(Object compiledScript, SearchLookup lookup, @Nullable Map vars) { Expression expr = (Expression)compiledScript; MapperService mapper = lookup.doc().mapperService(); - ExpressionScriptBindings bindings = new ExpressionScriptBindings(); + // NOTE: if we need to do anything complicated with bindings in the future, we can just extend Bindings, + // instead of complicating SimpleBindings (which should stay simple) + XSimpleBindings bindings = new XSimpleBindings(); for (String variable : expr.variables) { if (variable.equals("_score")) { - // noop: our bindings inherently know how to deal with score + bindings.add(new SortField("_score", SortField.Type.SCORE)); } else if (vars != null && vars.containsKey(variable)) { // TODO: document and/or error if vars contains _score? // NOTE: by checking for the variable in vars first, it allows masking document fields with a global constant, // but if we were to reverse it, we could provide a way to supply dynamic defaults for documents missing the field? Object value = vars.get(variable); if (value instanceof Double) { - bindings.addConstant(variable, ((Double)value).doubleValue()); + bindings.add(variable, new DoubleConstValueSource(((Double)value).doubleValue())); } else if (value instanceof Long) { - bindings.addConstant(variable, ((Long)value).doubleValue()); + bindings.add(variable, new DoubleConstValueSource(((Long)value).doubleValue())); } else if (value instanceof Integer) { - bindings.addConstant(variable, ((Integer)value).doubleValue()); + bindings.add(variable, new DoubleConstValueSource(((Integer)value).doubleValue())); } else { - throw new ExpressionScriptExecutionException("Parameter [" + variable + "] must be a numeric type (double or long)"); + throw new ExpressionScriptExecutionException("Parameter [" + variable + "] must be a numeric type"); } } else { - // TODO: extract field name/access pattern from variable - FieldMapper field = mapper.smartNameFieldMapper(variable); + XVariableContext[] parts = XVariableContext.parse(variable); + if (parts[0].text.equals("doc") == false) { + throw new ExpressionScriptExecutionException("Unknown variable [" + parts[0].text + "] in expression"); + } + if (parts.length < 2 || parts[1].type != XVariableContext.Type.STR_INDEX) { + throw new ExpressionScriptExecutionException("Variable 'doc' in expression must be used with a specific field like: doc['myfield'].value"); + } + if (parts.length < 3 || parts[2].type != XVariableContext.Type.MEMBER || parts[2].text.equals("value") == false) { + throw new ExpressionScriptExecutionException("Invalid member for field data in expression. Only '.value' is currently supported."); + } + String fieldname = parts[1].text; + + FieldMapper field = mapper.smartNameFieldMapper(fieldname); if (field.isNumeric() == false) { // TODO: more context (which expression?) throw new ExpressionScriptExecutionException("Field [" + variable + "] used in expression must be numeric"); } IndexFieldData fieldData = lookup.doc().fieldDataService.getForField((NumberFieldMapper)field); - bindings.addField(variable, fieldData); + bindings.add(variable, new ExpressionScriptValueSource(fieldData)); } } diff --git a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptExecutionException.java b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptExecutionException.java index 5d2cd3da578c6..66ff30b9e139f 100644 --- a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptExecutionException.java +++ b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptExecutionException.java @@ -1,3 +1,22 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.elasticsearch.script.expression; import org.elasticsearch.ElasticsearchException; diff --git a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptFunctionValues.java b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptFunctionValues.java index 9dd7899259c41..8baa82b5c741f 100644 --- a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptFunctionValues.java +++ b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptFunctionValues.java @@ -1,12 +1,32 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.elasticsearch.script.expression; -import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.docvalues.DoubleDocValues; import org.elasticsearch.index.fielddata.AtomicNumericFieldData; import org.elasticsearch.index.fielddata.DoubleValues; - +/** + * A {@link org.apache.lucene.queries.function.FunctionValues} which wrap field data. + */ class ExpressionScriptFunctionValues extends DoubleDocValues { DoubleValues dataAccessor; @@ -17,9 +37,10 @@ class ExpressionScriptFunctionValues extends DoubleDocValues { @Override public double doubleVal(int i) { - dataAccessor.setDocument(i); - // TODO: how are default values handled (if doc doesn't have value for this field?) - // TODO: how to handle nth value for array access in the future? + int numValues = dataAccessor.setDocument(i); + if (numValues == 0) { + return 0.0; + } return dataAccessor.nextValue(); } } diff --git a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptValueSource.java b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptValueSource.java index 495a9ccb96310..e1584004db797 100644 --- a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptValueSource.java +++ b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptValueSource.java @@ -1,3 +1,22 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + package org.elasticsearch.script.expression; @@ -11,6 +30,9 @@ import java.io.IOException; import java.util.Map; +/** + * A {@link ValueSource} wrapper for field data. + */ class ExpressionScriptValueSource extends ValueSource { IndexFieldData fieldData; diff --git a/src/test/java/org/elasticsearch/script/ExpressionScriptTest.java b/src/test/java/org/elasticsearch/script/ExpressionScriptTest.java deleted file mode 100644 index f09eb54009102..0000000000000 --- a/src/test/java/org/elasticsearch/script/ExpressionScriptTest.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.elasticsearch.script; - -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.test.ElasticsearchIntegrationTest; -import org.junit.Test; - -public class ExpressionScriptTest extends ElasticsearchIntegrationTest { - - @Test - public void testBasic() { - client().prepareIndex("test", "doc", "1").setSource("foo", 5).setRefresh(true).get(); - String script = "_score + foo + 2"; - SearchResponse resp = client().prepareSearch("test") - .setSource("{query: {" + - "function_score: {" + - "query:{match_all:{}}," + - "boost_mode: \"replace\"," + - "script_score: {" + - "script: \"" + script +"\"," + - "lang: \"expression\"" + - "}" + - "}" + - "}}").get(); - - assertEquals(8.0, resp.getHits().getAt(0).getScore(), 0.0001f); - } - - -} diff --git a/src/test/java/org/elasticsearch/script/expression/ExpressionScriptTests.java b/src/test/java/org/elasticsearch/script/expression/ExpressionScriptTests.java new file mode 100644 index 0000000000000..fee1c983d52cc --- /dev/null +++ b/src/test/java/org/elasticsearch/script/expression/ExpressionScriptTests.java @@ -0,0 +1,186 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.expression; + +import org.apache.lucene.expressions.Expression; +import org.apache.lucene.expressions.js.XJavascriptCompiler; +import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.action.search.SearchPhaseExecutionException; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.test.ElasticsearchIntegrationTest; + +import static org.hamcrest.Matchers.equalTo; + +public class ExpressionScriptTests extends ElasticsearchIntegrationTest { + + private SearchResponse runScript(String script, Object... params) { + StringBuilder buf = new StringBuilder(); + buf.append("{query: {match_all:{}}, sort: [{\"_uid\": {\"order\":\"asc\"}}], script_fields: {foo: {lang: \"expression\", script: \"" + script + "\""); + if (params.length > 0) { + assert(params.length % 2 == 0); + buf.append(",params:{"); + for (int i = 0; i < params.length; i += 2) { + if (i != 0) buf.append(","); + buf.append("\"" + params[i] + "\":"); + Object v = params[i + 1]; + if (v instanceof String) { + buf.append("\"" + v + "\""); + } else { + buf.append(v); + } + } + buf.append("}"); + } + buf.append("}}}"); + return client().prepareSearch("test").setSource(buf.toString()).get(); + } + + public void testBasic() throws Exception { + Expression x = XJavascriptCompiler.compile("1 + 1"); + System.out.println(x.sourceText); + client().prepareIndex("test", "doc", "1").setSource("foo", 4).setRefresh(true).get(); + SearchResponse rsp = runScript("doc['foo'].value + 1"); + assertEquals(1, rsp.getHits().getTotalHits()); + assertEquals(5.0, rsp.getHits().getAt(0).field("foo").getValue()); + } + + public void testScore() throws Exception { + client().prepareIndex("test", "doc", "1").setSource("text", "hello goodbye").get(); + client().prepareIndex("test", "doc", "2").setSource("text", "hello hello hello goodbye").get(); + client().prepareIndex("test", "doc", "3").setSource("text", "hello hello goodebye").get(); + refresh(); + String req = "{query: {function_score: {query:{term:{text:\"hello\"}}," + + "boost_mode: \"replace\"," + + "script_score: {" + + "script: \"1 / _score\"," + // invert the order given by score + "lang: \"expression\"" + + "}}}}"; + SearchResponse rsp = client().prepareSearch("test").setSource(req).get(); + SearchHits hits = rsp.getHits(); + assertEquals(3, hits.getTotalHits()); + assertEquals("1", hits.getAt(0).getId()); + assertEquals("3", hits.getAt(1).getId()); + assertEquals("2", hits.getAt(2).getId()); + } + + public void testFieldMissing() { + client().prepareIndex("test", "doc", "1").setSource("x", 4).get(); + client().prepareIndex("test", "doc", "2").setSource("y", 2).get(); + refresh(); + SearchResponse rsp = runScript("doc['x'].value + 1"); + SearchHits hits = rsp.getHits(); + assertEquals(2, rsp.getHits().getTotalHits()); + assertEquals(5.0, hits.getAt(0).field("foo").getValue()); + assertEquals(1.0, hits.getAt(1).field("foo").getValue()); + } + + public void testParams() { + client().prepareIndex("test", "doc", "1").setSource("x", 10).get(); + client().prepareIndex("test", "doc", "2").setSource("x", 3).get(); + client().prepareIndex("test", "doc", "3").setSource("x", 5).get(); + refresh(); + // a = int, b = double, c = long + SearchResponse rsp = runScript("doc['x'].value * a + b + ((c + doc['x'].value) > 5000000009 ? 1 : 0)", "a", 2, "b", 3.5, "c", 5000000000L); + SearchHits hits = rsp.getHits(); + assertEquals(3, hits.getTotalHits()); + assertEquals(24.5, hits.getAt(0).field("foo").getValue()); + assertEquals(9.5, hits.getAt(1).field("foo").getValue()); + assertEquals(13.5, hits.getAt(2).field("foo").getValue()); + } + + public void testCompileFailure() { + client().prepareIndex("test", "doc", "1").setSource("x", 1).setRefresh(true).get(); + try { + runScript("garbage%@#%@"); + fail("Expected expression compilation failure"); + } catch (SearchPhaseExecutionException e) { + assertThat(ExceptionsHelper.detailedMessage(e) + "should have contained ExpressionScriptCompilationException", + ExceptionsHelper.detailedMessage(e).contains("ExpressionScriptCompilationException"), equalTo(true)); + assertThat(ExceptionsHelper.detailedMessage(e) + "should have contained compilation failure", + ExceptionsHelper.detailedMessage(e).contains("Failed to parse expression"), equalTo(true)); + } + } + + public void testNonNumericParam() { + client().prepareIndex("test", "doc", "1").setSource("x", 1).setRefresh(true).get(); + try { + runScript("a", "a", "astring"); + fail("Expected string parameter to cause failure"); + } catch (SearchPhaseExecutionException e) { + assertThat(ExceptionsHelper.detailedMessage(e) + "should have contained ExpressionScriptExecutiontException", + ExceptionsHelper.detailedMessage(e).contains("ExpressionScriptExecutionException"), equalTo(true)); + assertThat(ExceptionsHelper.detailedMessage(e) + "should have contained non-numeric parameter error", + ExceptionsHelper.detailedMessage(e).contains("must be a numeric type"), equalTo(true)); + } + } + + public void testNonNumericField() { + client().prepareIndex("test", "doc", "1").setSource("text", "this is not a number").setRefresh(true).get(); + try { + runScript("doc['text'].value"); + fail("Expected text field to cause execution failure"); + } catch (SearchPhaseExecutionException e) { + assertThat(ExceptionsHelper.detailedMessage(e) + "should have contained ExpressionScriptExecutiontException", + ExceptionsHelper.detailedMessage(e).contains("ExpressionScriptExecutionException"), equalTo(true)); + assertThat(ExceptionsHelper.detailedMessage(e) + "should have contained non-numeric field error", + ExceptionsHelper.detailedMessage(e).contains("must be numeric"), equalTo(true)); + } + } + + public void testInvalidGlobalVariable() { + client().prepareIndex("test", "doc", "1").setSource("foo", 5).setRefresh(true).get(); + try { + runScript("bogus"); + fail("Expected bogus variable to cause execution failure"); + } catch (SearchPhaseExecutionException e) { + assertThat(ExceptionsHelper.detailedMessage(e) + "should have contained ExpressionScriptExecutiontException", + ExceptionsHelper.detailedMessage(e).contains("ExpressionScriptExecutionException"), equalTo(true)); + assertThat(ExceptionsHelper.detailedMessage(e) + "should have contained unknown variable error", + ExceptionsHelper.detailedMessage(e).contains("Unknown variable"), equalTo(true)); + } + } + + public void testDocWithoutField() { + client().prepareIndex("test", "doc", "1").setSource("foo", 5).setRefresh(true).get(); + try { + runScript("doc"); + fail("Expected doc variable without field to cause execution failure"); + } catch (SearchPhaseExecutionException e) { + assertThat(ExceptionsHelper.detailedMessage(e) + "should have contained ExpressionScriptExecutiontException", + ExceptionsHelper.detailedMessage(e).contains("ExpressionScriptExecutionException"), equalTo(true)); + assertThat(ExceptionsHelper.detailedMessage(e) + "should have contained a missing specific field error", + ExceptionsHelper.detailedMessage(e).contains("must be used with a specific field"), equalTo(true)); + } + } + + public void testInvalidFieldMember() { + client().prepareIndex("test", "doc", "1").setSource("foo", 5).setRefresh(true).get(); + try { + runScript("doc['foo'].bogus"); + fail("Expected bogus field member to cause execution failure"); + } catch (SearchPhaseExecutionException e) { + assertThat(ExceptionsHelper.detailedMessage(e) + "should have contained ExpressionScriptExecutiontException", + ExceptionsHelper.detailedMessage(e).contains("ExpressionScriptExecutionException"), equalTo(true)); + assertThat(ExceptionsHelper.detailedMessage(e) + "should have contained field member error", + ExceptionsHelper.detailedMessage(e).contains("Invalid member for field"), equalTo(true)); + } + } +} From c3726a84e3069437f607e82994a940222c2aad1a Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 10 Jul 2014 08:35:58 -0700 Subject: [PATCH 05/16] Fixed possible NPE when mapping doesn't exist and added test --- .../ExpressionScriptEngineService.java | 5 ++- .../expression/ExpressionScriptTests.java | 32 ++++++++++++++++--- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java index 2b40e44432b68..217afb725c968 100644 --- a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java +++ b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java @@ -116,9 +116,12 @@ public SearchScript search(Object compiledScript, SearchLookup lookup, @Nullable String fieldname = parts[1].text; FieldMapper field = mapper.smartNameFieldMapper(fieldname); + if (field == null) { + throw new ExpressionScriptExecutionException("Field [" + fieldname + "] used in expression does not exist in mappings"); + } if (field.isNumeric() == false) { // TODO: more context (which expression?) - throw new ExpressionScriptExecutionException("Field [" + variable + "] used in expression must be numeric"); + throw new ExpressionScriptExecutionException("Field [" + fieldname + "] used in expression must be numeric"); } IndexFieldData fieldData = lookup.doc().fieldDataService.getForField((NumberFieldMapper)field); bindings.add(variable, new ExpressionScriptValueSource(fieldData)); diff --git a/src/test/java/org/elasticsearch/script/expression/ExpressionScriptTests.java b/src/test/java/org/elasticsearch/script/expression/ExpressionScriptTests.java index fee1c983d52cc..c9c5a5fcca780 100644 --- a/src/test/java/org/elasticsearch/script/expression/ExpressionScriptTests.java +++ b/src/test/java/org/elasticsearch/script/expression/ExpressionScriptTests.java @@ -19,19 +19,19 @@ package org.elasticsearch.script.expression; -import org.apache.lucene.expressions.Expression; -import org.apache.lucene.expressions.js.XJavascriptCompiler; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.search.SearchHits; import org.elasticsearch.test.ElasticsearchIntegrationTest; +import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; import static org.hamcrest.Matchers.equalTo; public class ExpressionScriptTests extends ElasticsearchIntegrationTest { private SearchResponse runScript(String script, Object... params) { + ensureGreen("test"); StringBuilder buf = new StringBuilder(); buf.append("{query: {match_all:{}}, sort: [{\"_uid\": {\"order\":\"asc\"}}], script_fields: {foo: {lang: \"expression\", script: \"" + script + "\""); if (params.length > 0) { @@ -54,8 +54,8 @@ private SearchResponse runScript(String script, Object... params) { } public void testBasic() throws Exception { - Expression x = XJavascriptCompiler.compile("1 + 1"); - System.out.println(x.sourceText); + createIndex("test"); + ensureGreen("test"); client().prepareIndex("test", "doc", "1").setSource("foo", 4).setRefresh(true).get(); SearchResponse rsp = runScript("doc['foo'].value + 1"); assertEquals(1, rsp.getHits().getTotalHits()); @@ -63,6 +63,8 @@ public void testBasic() throws Exception { } public void testScore() throws Exception { + createIndex("test"); + ensureGreen("test"); client().prepareIndex("test", "doc", "1").setSource("text", "hello goodbye").get(); client().prepareIndex("test", "doc", "2").setSource("text", "hello hello hello goodbye").get(); client().prepareIndex("test", "doc", "3").setSource("text", "hello hello goodebye").get(); @@ -81,18 +83,38 @@ public void testScore() throws Exception { assertEquals("2", hits.getAt(2).getId()); } - public void testFieldMissing() { + public void testSparseField() throws Exception { + ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "x", "type=long", "y", "type=long")); + ensureGreen("test"); client().prepareIndex("test", "doc", "1").setSource("x", 4).get(); client().prepareIndex("test", "doc", "2").setSource("y", 2).get(); refresh(); SearchResponse rsp = runScript("doc['x'].value + 1"); + ElasticsearchAssertions.assertSearchResponse(rsp); SearchHits hits = rsp.getHits(); assertEquals(2, rsp.getHits().getTotalHits()); assertEquals(5.0, hits.getAt(0).field("foo").getValue()); assertEquals(1.0, hits.getAt(1).field("foo").getValue()); } + public void testMissingField() throws Exception { + createIndex("test"); + ensureGreen("test"); + client().prepareIndex("test", "doc", "1").setSource("x", 4).setRefresh(true).get(); + try { + runScript("doc['bogus'].value"); + fail("Expected missing field to cause failure"); + } catch (SearchPhaseExecutionException e) { + assertThat(ExceptionsHelper.detailedMessage(e) + "should have contained ExpressionScriptExecutionException", + ExceptionsHelper.detailedMessage(e).contains("ExpressionScriptExecutionException"), equalTo(true)); + assertThat(ExceptionsHelper.detailedMessage(e) + "should have contained missing field error", + ExceptionsHelper.detailedMessage(e).contains("does not exist in mappings"), equalTo(true)); + } + } + public void testParams() { + createIndex("test"); + ensureGreen("test"); client().prepareIndex("test", "doc", "1").setSource("x", 10).get(); client().prepareIndex("test", "doc", "2").setSource("x", 3).get(); client().prepareIndex("test", "doc", "3").setSource("x", 5).get(); From b8c8644b0b5376ffa0efdc7ab8d2f6a8320ef05f Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 10 Jul 2014 08:43:39 -0700 Subject: [PATCH 06/16] Add assert to trip on upgrade to 4.10 --- .../java/org/apache/lucene/expressions/XSimpleBindings.java | 5 +++++ .../apache/lucene/expressions/js/XJavascriptCompiler.java | 4 ++++ .../org/apache/lucene/expressions/js/XJavascriptLexer.java | 5 +++++ .../org/apache/lucene/expressions/js/XJavascriptParser.java | 5 +++++ .../org/apache/lucene/expressions/js/XVariableContext.java | 5 +++++ 5 files changed, 24 insertions(+) diff --git a/src/main/java/org/apache/lucene/expressions/XSimpleBindings.java b/src/main/java/org/apache/lucene/expressions/XSimpleBindings.java index d72177e2bd1ea..8a6ac2847af78 100644 --- a/src/main/java/org/apache/lucene/expressions/XSimpleBindings.java +++ b/src/main/java/org/apache/lucene/expressions/XSimpleBindings.java @@ -48,6 +48,11 @@ * @lucene.experimental */ public final class XSimpleBindings extends Bindings { + + static { + assert org.elasticsearch.Version.CURRENT.luceneVersion == org.apache.lucene.util.Version.LUCENE_4_9: "Remove this code once we upgrade to Lucene 4.10 (LUCENE-5806)"; + } + final Map map = new HashMap<>(); /** Creates a new empty Bindings */ diff --git a/src/main/java/org/apache/lucene/expressions/js/XJavascriptCompiler.java b/src/main/java/org/apache/lucene/expressions/js/XJavascriptCompiler.java index 60afa9efbbbce..5dfb52c744ad7 100644 --- a/src/main/java/org/apache/lucene/expressions/js/XJavascriptCompiler.java +++ b/src/main/java/org/apache/lucene/expressions/js/XJavascriptCompiler.java @@ -74,6 +74,10 @@ */ public class XJavascriptCompiler { + static { + assert org.elasticsearch.Version.CURRENT.luceneVersion == org.apache.lucene.util.Version.LUCENE_4_9: "Remove this code once we upgrade to Lucene 4.10 (LUCENE-5806)"; + } + static final class Loader extends ClassLoader { Loader(ClassLoader parent) { super(parent); diff --git a/src/main/java/org/apache/lucene/expressions/js/XJavascriptLexer.java b/src/main/java/org/apache/lucene/expressions/js/XJavascriptLexer.java index f63a98dd4c522..5757c4d7b30bb 100644 --- a/src/main/java/org/apache/lucene/expressions/js/XJavascriptLexer.java +++ b/src/main/java/org/apache/lucene/expressions/js/XJavascriptLexer.java @@ -12,6 +12,11 @@ @SuppressWarnings("all") class XJavascriptLexer extends Lexer { + + static { + assert org.elasticsearch.Version.CURRENT.luceneVersion == org.apache.lucene.util.Version.LUCENE_4_9: "Remove this code once we upgrade to Lucene 4.10 (LUCENE-5806)"; + } + public static final int EOF=-1; public static final int ARRAY=4; public static final int AT_ADD=5; diff --git a/src/main/java/org/apache/lucene/expressions/js/XJavascriptParser.java b/src/main/java/org/apache/lucene/expressions/js/XJavascriptParser.java index 433dae4000a61..04fd0f5f4c9f6 100644 --- a/src/main/java/org/apache/lucene/expressions/js/XJavascriptParser.java +++ b/src/main/java/org/apache/lucene/expressions/js/XJavascriptParser.java @@ -15,6 +15,11 @@ @SuppressWarnings("all") class XJavascriptParser extends Parser { + + static { + assert org.elasticsearch.Version.CURRENT.luceneVersion == org.apache.lucene.util.Version.LUCENE_4_9: "Remove this code once we upgrade to Lucene 4.10 (LUCENE-5806)"; + } + public static final String[] tokenNames = new String[] { "", "", "", "", "ARRAY", "AT_ADD", "AT_BIT_AND", "AT_BIT_NOT", "AT_BIT_OR", "AT_BIT_SHL", "AT_BIT_SHR", "AT_BIT_SHU", "AT_BIT_XOR", diff --git a/src/main/java/org/apache/lucene/expressions/js/XVariableContext.java b/src/main/java/org/apache/lucene/expressions/js/XVariableContext.java index fa0382a1e0995..5b1b342d1e22f 100644 --- a/src/main/java/org/apache/lucene/expressions/js/XVariableContext.java +++ b/src/main/java/org/apache/lucene/expressions/js/XVariableContext.java @@ -25,6 +25,11 @@ * sequence of array (integer or string indexed) and member accesses. */ public class XVariableContext { + + static { + assert org.elasticsearch.Version.CURRENT.luceneVersion == org.apache.lucene.util.Version.LUCENE_4_9: "Remove this code once we upgrade to Lucene 4.10 (LUCENE-5806)"; + } + public static enum Type { MEMBER, // "dot" access STR_INDEX, // brackets with a string From 631a38d5a9e5796f56e00c7c57f8d7ba911c7680 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 10 Jul 2014 10:39:39 -0700 Subject: [PATCH 07/16] Test terms aggs fail with expressions --- .../script/expression/ExpressionScript.java | 16 ++++- .../ExpressionScriptEngineService.java | 4 ++ .../expression/ExpressionScriptTests.java | 63 +++++++++++++++++++ 3 files changed, 80 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/elasticsearch/script/expression/ExpressionScript.java b/src/main/java/org/elasticsearch/script/expression/ExpressionScript.java index 559430c58eb53..139a421f0a4aa 100644 --- a/src/main/java/org/elasticsearch/script/expression/ExpressionScript.java +++ b/src/main/java/org/elasticsearch/script/expression/ExpressionScript.java @@ -21,9 +21,11 @@ import org.apache.lucene.expressions.Bindings; import org.apache.lucene.expressions.Expression; +import org.apache.lucene.expressions.XSimpleBindings; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.ValueSource; +import org.apache.lucene.queries.function.valuesource.DoubleConstValueSource; import org.apache.lucene.search.Scorer; import org.elasticsearch.script.SearchScript; @@ -78,11 +80,11 @@ public long cost() { } Expression expression; - Bindings bindings; + XSimpleBindings bindings; AtomicReaderContext leaf; CannedScorer scorer; - ExpressionScript(Expression e, Bindings b) { + ExpressionScript(Expression e, XSimpleBindings b) { expression = e; bindings = b; scorer = new CannedScorer(); @@ -136,7 +138,15 @@ public void setNextSource(Map source) { @Override public void setNextVar(String name, Object value) { - // nono commit: comment on why this isn't needed...wtf is it? + // this assumes that the same variable will be set for every document evaluated, thus + // the variable never needs to be removed from the bindings, but only overwritten + if (value instanceof Double) { + bindings.add(name, new DoubleConstValueSource(((Double)value).doubleValue())); + } else if (value instanceof Long) { + bindings.add(name, new DoubleConstValueSource(((Long)value).doubleValue())); + } else { + throw new ExpressionScriptExecutionException("Cannot use expression with text variable"); + } } diff --git a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java index 217afb725c968..cfc53ef8c2ad9 100644 --- a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java +++ b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java @@ -88,6 +88,10 @@ public SearchScript search(Object compiledScript, SearchLookup lookup, @Nullable for (String variable : expr.variables) { if (variable.equals("_score")) { bindings.add(new SortField("_score", SortField.Type.SCORE)); + } else if (variable.equals("_value")) { + // noop: _value is special for aggregations, and is added to bindings dynamically + // TODO: if some uses it in a scoring expression, they will get a nasty failure when evaluating...need a + // way to know this is for aggregations and so _value is ok to have... } else if (vars != null && vars.containsKey(variable)) { // TODO: document and/or error if vars contains _score? // NOTE: by checking for the variable in vars first, it allows masking document fields with a global constant, diff --git a/src/test/java/org/elasticsearch/script/expression/ExpressionScriptTests.java b/src/test/java/org/elasticsearch/script/expression/ExpressionScriptTests.java index c9c5a5fcca780..20abfda7d7169 100644 --- a/src/test/java/org/elasticsearch/script/expression/ExpressionScriptTests.java +++ b/src/test/java/org/elasticsearch/script/expression/ExpressionScriptTests.java @@ -23,10 +23,13 @@ import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.aggregations.metrics.stats.Stats; import org.elasticsearch.test.ElasticsearchIntegrationTest; import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; +import org.elasticsearch.test.junit.annotations.TestLogging; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; public class ExpressionScriptTests extends ElasticsearchIntegrationTest { @@ -205,4 +208,64 @@ public void testInvalidFieldMember() { ExceptionsHelper.detailedMessage(e).contains("Invalid member for field"), equalTo(true)); } } + + public void testSpecialValueVariable() throws Exception { + // i.e. _value for aggregations + createIndex("test"); + ensureGreen("test"); + client().prepareIndex("test", "doc", "1").setSource("x", 5, "y", 1.2).get(); + client().prepareIndex("test", "doc", "2").setSource("x", 10, "y", 1.4).get(); + client().prepareIndex("test", "doc", "3").setSource("x", 13, "y", 1.8).get(); + refresh(); + String req = "{query: {match_all:{}}, aggs: {" + + "int_agg: {stats: {" + + "field: \"x\"," + + "script: \"_value * 3\"," + + "lang: \"expression\"" + + "}},double_agg: {stats: {" + + "field: \"y\"," + + "script: \"_value - 1.1\"," + + "lang: \"expression\"" + + "}}}}"; + SearchResponse rsp = client().prepareSearch("test").setSource(req).get(); + assertEquals(3, rsp.getHits().getTotalHits()); + + Stats stats = rsp.getAggregations().get("int_agg"); + assertEquals(39.0, stats.getMax(), 0.0001); + assertEquals(15.0, stats.getMin(), 0.0001); + + stats = rsp.getAggregations().get("double_agg"); + assertEquals(0.7, stats.getMax(), 0.0001); + assertEquals(0.1, stats.getMin(), 0.0001); + } + + public void testStringSpecialValueVariable() throws Exception { + // i.e. expression script for term aggregations, which is not allowed + createIndex("test"); + ensureGreen("test"); + client().prepareIndex("test", "doc", "1").setSource("text", "hello").get(); + client().prepareIndex("test", "doc", "2").setSource("text", "goodbye").get(); + client().prepareIndex("test", "doc", "3").setSource("text", "hello").get(); + refresh(); + String req = "{query: {match_all:{}}, aggs: {term_agg: {terms: {" + + "field: \"text\"," + + "script: \"_value\"," + + "lang: \"expression\"" + + "}}}}"; + + Throwable t; + try { + // shards that don't have docs with the "text" field will not fail, + // so we may or may not get a total failure + SearchResponse rsp = client().prepareSearch("test").setSource(req).get(); + assertThat(rsp.getShardFailures().length, greaterThan(0)); // at least the shards containing the docs should have failed + t = rsp.getShardFailures()[0].failure(); + } catch (SearchPhaseExecutionException e) { + t = e; + } + assertThat(ExceptionsHelper.detailedMessage(t) + "should have contained ExpressionScriptExecutiontException", + ExceptionsHelper.detailedMessage(t).contains("ExpressionScriptExecutionException"), equalTo(true)); + assertThat(ExceptionsHelper.detailedMessage(t) + "should have contained text variable error", + ExceptionsHelper.detailedMessage(t).contains("text variable"), equalTo(true)); + } } From 8edb0ce11c425afd182e1d95682ebcd0965c8300 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 10 Jul 2014 11:18:31 -0700 Subject: [PATCH 08/16] Minor readability tweaks before PR --- .../script/expression/ExpressionScriptEngineService.java | 3 +++ .../script/expression/ExpressionScriptFunctionValues.java | 1 + 2 files changed, 4 insertions(+) diff --git a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java index cfc53ef8c2ad9..1f6488b1882df 100644 --- a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java +++ b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java @@ -88,10 +88,12 @@ public SearchScript search(Object compiledScript, SearchLookup lookup, @Nullable for (String variable : expr.variables) { if (variable.equals("_score")) { bindings.add(new SortField("_score", SortField.Type.SCORE)); + } else if (variable.equals("_value")) { // noop: _value is special for aggregations, and is added to bindings dynamically // TODO: if some uses it in a scoring expression, they will get a nasty failure when evaluating...need a // way to know this is for aggregations and so _value is ok to have... + } else if (vars != null && vars.containsKey(variable)) { // TODO: document and/or error if vars contains _score? // NOTE: by checking for the variable in vars first, it allows masking document fields with a global constant, @@ -106,6 +108,7 @@ public SearchScript search(Object compiledScript, SearchLookup lookup, @Nullable } else { throw new ExpressionScriptExecutionException("Parameter [" + variable + "] must be a numeric type"); } + } else { XVariableContext[] parts = XVariableContext.parse(variable); if (parts[0].text.equals("doc") == false) { diff --git a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptFunctionValues.java b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptFunctionValues.java index 8baa82b5c741f..c2122eee80627 100644 --- a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptFunctionValues.java +++ b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptFunctionValues.java @@ -39,6 +39,7 @@ class ExpressionScriptFunctionValues extends DoubleDocValues { public double doubleVal(int i) { int numValues = dataAccessor.setDocument(i); if (numValues == 0) { + // sparse fields get a value of 0 when the field doesn't exist return 0.0; } return dataAccessor.nextValue(); From 8a31ae101a71d297a9880cc1668969c39bc5cf43 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 10 Jul 2014 14:24:38 -0700 Subject: [PATCH 09/16] Address first round of review comments --- .../script/expression/ExpressionScript.java | 44 +++++++----- .../ExpressionScriptEngineService.java | 22 +++--- ...lues.java => FieldDataFunctionValues.java} | 4 +- ...eSource.java => FieldDataValueSource.java} | 6 +- .../ReplaceableConstValueSource.java | 69 +++++++++++++++++++ .../expression/ExpressionScriptTests.java | 41 ++++++----- 6 files changed, 129 insertions(+), 57 deletions(-) rename src/main/java/org/elasticsearch/script/expression/{ExpressionScriptFunctionValues.java => FieldDataFunctionValues.java} (91%) rename src/main/java/org/elasticsearch/script/expression/{ExpressionScriptValueSource.java => FieldDataValueSource.java} (90%) create mode 100644 src/main/java/org/elasticsearch/script/expression/ReplaceableConstValueSource.java diff --git a/src/main/java/org/elasticsearch/script/expression/ExpressionScript.java b/src/main/java/org/elasticsearch/script/expression/ExpressionScript.java index 139a421f0a4aa..ef3376c4e7239 100644 --- a/src/main/java/org/elasticsearch/script/expression/ExpressionScript.java +++ b/src/main/java/org/elasticsearch/script/expression/ExpressionScript.java @@ -25,7 +25,6 @@ import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.ValueSource; -import org.apache.lucene.queries.function.valuesource.DoubleConstValueSource; import org.apache.lucene.search.Scorer; import org.elasticsearch.script.SearchScript; @@ -81,23 +80,23 @@ public long cost() { Expression expression; XSimpleBindings bindings; - AtomicReaderContext leaf; CannedScorer scorer; + ValueSource source; + FunctionValues values; + Map context; + ReplaceableConstValueSource specialValue; // _value - ExpressionScript(Expression e, XSimpleBindings b) { + ExpressionScript(Expression e, XSimpleBindings b, ReplaceableConstValueSource v) { expression = e; bindings = b; scorer = new CannedScorer(); + context = Collections.singletonMap("scorer", scorer); + source = expression.getValueSource(bindings); + specialValue = v; } double evaluate() { - try { - ValueSource vs = expression.getValueSource(bindings); - FunctionValues fv = vs.getValues(Collections.singletonMap("scorer", scorer), leaf); - return fv.doubleVal(scorer.docid); - } catch (IOException e) { - throw new ExpressionScriptExecutionException("Failed to run expression", e); - } + return values.doubleVal(scorer.docid); } @Override @@ -124,12 +123,19 @@ public void setNextDocId(int d) { public void setNextScore(float score) { scorer.score = score; } @Override - public void setNextReader(AtomicReaderContext l) { - leaf = l; + public void setNextReader(AtomicReaderContext leaf) { + try { + values = source.getValues(context, leaf); + } catch (IOException e) { + throw new ExpressionScriptExecutionException("Failed to run expression", e); + } } @Override - public void setScorer(Scorer s) { /* noop: score isn't actually set for scoring... */ } + public void setScorer(Scorer s) { + // noop: The scorer isn't actually ever set. Instead setNextScore is called. + // NOTE: This seems broken. Why can't we just use the scorer and get rid of setNextScore? + } @Override public void setNextSource(Map source) { @@ -138,12 +144,12 @@ public void setNextSource(Map source) { @Override public void setNextVar(String name, Object value) { - // this assumes that the same variable will be set for every document evaluated, thus - // the variable never needs to be removed from the bindings, but only overwritten - if (value instanceof Double) { - bindings.add(name, new DoubleConstValueSource(((Double)value).doubleValue())); - } else if (value instanceof Long) { - bindings.add(name, new DoubleConstValueSource(((Long)value).doubleValue())); + assert(specialValue != null); + // this should only be used for the special "_value" variable used in aggregations + assert(name.equals("_value")); + + if (value instanceof Number) { + specialValue.setValue(((Number)value).doubleValue()); } else { throw new ExpressionScriptExecutionException("Cannot use expression with text variable"); } diff --git a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java index 1f6488b1882df..f8633c70ebd0a 100644 --- a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java +++ b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java @@ -84,13 +84,16 @@ public SearchScript search(Object compiledScript, SearchLookup lookup, @Nullable // NOTE: if we need to do anything complicated with bindings in the future, we can just extend Bindings, // instead of complicating SimpleBindings (which should stay simple) XSimpleBindings bindings = new XSimpleBindings(); + ReplaceableConstValueSource specialValue = null; for (String variable : expr.variables) { if (variable.equals("_score")) { bindings.add(new SortField("_score", SortField.Type.SCORE)); } else if (variable.equals("_value")) { - // noop: _value is special for aggregations, and is added to bindings dynamically + specialValue = new ReplaceableConstValueSource(); + bindings.add("_value", specialValue); + // noop: _value is special for aggregations, and is handled in ExpressionScriptBindings // TODO: if some uses it in a scoring expression, they will get a nasty failure when evaluating...need a // way to know this is for aggregations and so _value is ok to have... @@ -99,12 +102,8 @@ public SearchScript search(Object compiledScript, SearchLookup lookup, @Nullable // NOTE: by checking for the variable in vars first, it allows masking document fields with a global constant, // but if we were to reverse it, we could provide a way to supply dynamic defaults for documents missing the field? Object value = vars.get(variable); - if (value instanceof Double) { - bindings.add(variable, new DoubleConstValueSource(((Double)value).doubleValue())); - } else if (value instanceof Long) { - bindings.add(variable, new DoubleConstValueSource(((Long)value).doubleValue())); - } else if (value instanceof Integer) { - bindings.add(variable, new DoubleConstValueSource(((Integer)value).doubleValue())); + if (value instanceof Number) { + bindings.add(variable, new DoubleConstValueSource(((Number)value).doubleValue())); } else { throw new ExpressionScriptExecutionException("Parameter [" + variable + "] must be a numeric type"); } @@ -131,22 +130,21 @@ public SearchScript search(Object compiledScript, SearchLookup lookup, @Nullable throw new ExpressionScriptExecutionException("Field [" + fieldname + "] used in expression must be numeric"); } IndexFieldData fieldData = lookup.doc().fieldDataService.getForField((NumberFieldMapper)field); - bindings.add(variable, new ExpressionScriptValueSource(fieldData)); + bindings.add(variable, new FieldDataValueSource(fieldData)); } } - return new ExpressionScript((Expression)compiledScript, bindings); + return new ExpressionScript((Expression)compiledScript, bindings, specialValue); } @Override public ExecutableScript executable(Object compiledScript, @Nullable Map vars) { - // cannot use expressions for updates (yet) - throw new UnsupportedOperationException(); + throw new UnsupportedOperationException("Cannot use expressions for updates"); } @Override public Object execute(Object compiledScript, Map vars) { - throw new UnsupportedOperationException(); + throw new UnsupportedOperationException("Cannot use expressions for updates"); } @Override diff --git a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptFunctionValues.java b/src/main/java/org/elasticsearch/script/expression/FieldDataFunctionValues.java similarity index 91% rename from src/main/java/org/elasticsearch/script/expression/ExpressionScriptFunctionValues.java rename to src/main/java/org/elasticsearch/script/expression/FieldDataFunctionValues.java index c2122eee80627..ed374908cd6c3 100644 --- a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptFunctionValues.java +++ b/src/main/java/org/elasticsearch/script/expression/FieldDataFunctionValues.java @@ -27,10 +27,10 @@ /** * A {@link org.apache.lucene.queries.function.FunctionValues} which wrap field data. */ -class ExpressionScriptFunctionValues extends DoubleDocValues { +class FieldDataFunctionValues extends DoubleDocValues { DoubleValues dataAccessor; - ExpressionScriptFunctionValues(ValueSource parent, AtomicNumericFieldData d) { + FieldDataFunctionValues(ValueSource parent, AtomicNumericFieldData d) { super(parent); dataAccessor = d.getDoubleValues(); } diff --git a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptValueSource.java b/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java similarity index 90% rename from src/main/java/org/elasticsearch/script/expression/ExpressionScriptValueSource.java rename to src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java index e1584004db797..a8c455bfe7840 100644 --- a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptValueSource.java +++ b/src/main/java/org/elasticsearch/script/expression/FieldDataValueSource.java @@ -33,11 +33,11 @@ /** * A {@link ValueSource} wrapper for field data. */ -class ExpressionScriptValueSource extends ValueSource { +class FieldDataValueSource extends ValueSource { IndexFieldData fieldData; - ExpressionScriptValueSource(IndexFieldData d) { + FieldDataValueSource(IndexFieldData d) { fieldData = d; } @@ -45,7 +45,7 @@ class ExpressionScriptValueSource extends ValueSource { public FunctionValues getValues(Map context, AtomicReaderContext leaf) throws IOException { AtomicFieldData leafData = fieldData.load(leaf); assert(leafData instanceof AtomicNumericFieldData); - return new ExpressionScriptFunctionValues(this, (AtomicNumericFieldData)leafData); + return new FieldDataFunctionValues(this, (AtomicNumericFieldData)leafData); } @Override diff --git a/src/main/java/org/elasticsearch/script/expression/ReplaceableConstValueSource.java b/src/main/java/org/elasticsearch/script/expression/ReplaceableConstValueSource.java new file mode 100644 index 0000000000000..d4168453b0f95 --- /dev/null +++ b/src/main/java/org/elasticsearch/script/expression/ReplaceableConstValueSource.java @@ -0,0 +1,69 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.script.expression; + +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.queries.function.FunctionValues; +import org.apache.lucene.queries.function.ValueSource; +import org.apache.lucene.queries.function.docvalues.DoubleDocValues; + +import java.io.IOException; +import java.util.Map; + +/** + * A {@link ValueSource} which has a stub {@link FunctionValues} that holds a dynamically replaceable constant double. + */ +class ReplaceableConstValueSource extends ValueSource { + double value; + final FunctionValues fv; + + public ReplaceableConstValueSource() { + fv = new DoubleDocValues(this) { + @Override + public double doubleVal(int i) { + return value; + } + }; + } + + @Override + public FunctionValues getValues(Map map, AtomicReaderContext atomicReaderContext) throws IOException { + return fv; + } + + @Override + public boolean equals(Object o) { + return o == this; + } + + @Override + public int hashCode() { + return System.identityHashCode(this); + } + + @Override + public String description() { + return "replaceableConstDouble"; + } + + public void setValue(double v) { + value = v; + } +} diff --git a/src/test/java/org/elasticsearch/script/expression/ExpressionScriptTests.java b/src/test/java/org/elasticsearch/script/expression/ExpressionScriptTests.java index 20abfda7d7169..1540f5457a46c 100644 --- a/src/test/java/org/elasticsearch/script/expression/ExpressionScriptTests.java +++ b/src/test/java/org/elasticsearch/script/expression/ExpressionScriptTests.java @@ -26,7 +26,6 @@ import org.elasticsearch.search.aggregations.metrics.stats.Stats; import org.elasticsearch.test.ElasticsearchIntegrationTest; import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; -import org.elasticsearch.test.junit.annotations.TestLogging; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -68,10 +67,10 @@ public void testBasic() throws Exception { public void testScore() throws Exception { createIndex("test"); ensureGreen("test"); - client().prepareIndex("test", "doc", "1").setSource("text", "hello goodbye").get(); - client().prepareIndex("test", "doc", "2").setSource("text", "hello hello hello goodbye").get(); - client().prepareIndex("test", "doc", "3").setSource("text", "hello hello goodebye").get(); - refresh(); + indexRandom(true, + client().prepareIndex("test", "doc", "1").setSource("text", "hello goodbye"), + client().prepareIndex("test", "doc", "2").setSource("text", "hello hello hello goodbye"), + client().prepareIndex("test", "doc", "3").setSource("text", "hello hello goodebye")); String req = "{query: {function_score: {query:{term:{text:\"hello\"}}," + "boost_mode: \"replace\"," + "script_score: {" + @@ -89,9 +88,9 @@ public void testScore() throws Exception { public void testSparseField() throws Exception { ElasticsearchAssertions.assertAcked(prepareCreate("test").addMapping("doc", "x", "type=long", "y", "type=long")); ensureGreen("test"); - client().prepareIndex("test", "doc", "1").setSource("x", 4).get(); - client().prepareIndex("test", "doc", "2").setSource("y", 2).get(); - refresh(); + indexRandom(true, + client().prepareIndex("test", "doc", "1").setSource("x", 4), + client().prepareIndex("test", "doc", "2").setSource("y", 2)); SearchResponse rsp = runScript("doc['x'].value + 1"); ElasticsearchAssertions.assertSearchResponse(rsp); SearchHits hits = rsp.getHits(); @@ -115,13 +114,13 @@ public void testMissingField() throws Exception { } } - public void testParams() { + public void testParams() throws Exception { createIndex("test"); ensureGreen("test"); - client().prepareIndex("test", "doc", "1").setSource("x", 10).get(); - client().prepareIndex("test", "doc", "2").setSource("x", 3).get(); - client().prepareIndex("test", "doc", "3").setSource("x", 5).get(); - refresh(); + indexRandom(true, + client().prepareIndex("test", "doc", "1").setSource("x", 10), + client().prepareIndex("test", "doc", "2").setSource("x", 3), + client().prepareIndex("test", "doc", "3").setSource("x", 5)); // a = int, b = double, c = long SearchResponse rsp = runScript("doc['x'].value * a + b + ((c + doc['x'].value) > 5000000009 ? 1 : 0)", "a", 2, "b", 3.5, "c", 5000000000L); SearchHits hits = rsp.getHits(); @@ -213,10 +212,10 @@ public void testSpecialValueVariable() throws Exception { // i.e. _value for aggregations createIndex("test"); ensureGreen("test"); - client().prepareIndex("test", "doc", "1").setSource("x", 5, "y", 1.2).get(); - client().prepareIndex("test", "doc", "2").setSource("x", 10, "y", 1.4).get(); - client().prepareIndex("test", "doc", "3").setSource("x", 13, "y", 1.8).get(); - refresh(); + indexRandom(true, + client().prepareIndex("test", "doc", "1").setSource("x", 5, "y", 1.2), + client().prepareIndex("test", "doc", "2").setSource("x", 10, "y", 1.4), + client().prepareIndex("test", "doc", "3").setSource("x", 13, "y", 1.8)); String req = "{query: {match_all:{}}, aggs: {" + "int_agg: {stats: {" + "field: \"x\"," + @@ -243,10 +242,10 @@ public void testStringSpecialValueVariable() throws Exception { // i.e. expression script for term aggregations, which is not allowed createIndex("test"); ensureGreen("test"); - client().prepareIndex("test", "doc", "1").setSource("text", "hello").get(); - client().prepareIndex("test", "doc", "2").setSource("text", "goodbye").get(); - client().prepareIndex("test", "doc", "3").setSource("text", "hello").get(); - refresh(); + indexRandom(true, + client().prepareIndex("test", "doc", "1").setSource("text", "hello"), + client().prepareIndex("test", "doc", "2").setSource("text", "goodbye"), + client().prepareIndex("test", "doc", "3").setSource("text", "hello")); String req = "{query: {match_all:{}}, aggs: {term_agg: {terms: {" + "field: \"text\"," + "script: \"_value\"," + From 5a42e4cbf4508826d861e03b52b355e9054a2de0 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Thu, 10 Jul 2014 15:02:53 -0700 Subject: [PATCH 10/16] Added scoring TODO and made expressions deps optional --- pom.xml | 3 ++- .../script/expression/ExpressionScript.java | 9 ++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 6aaf5401fa9d2..aa1b49b27b06b 100644 --- a/pom.xml +++ b/pom.xml @@ -154,6 +154,7 @@ lucene-expressions ${lucene.version} compile + true com.spatial4j @@ -879,7 +880,7 @@ ${project.build.directory}/lib - lucene*, log4j*, jna*, spatial4j*, jts*, groovy* + lucene*, log4j*, jna*, spatial4j*, jts*, groovy*, directory perm diff --git a/src/main/java/org/elasticsearch/script/expression/ExpressionScript.java b/src/main/java/org/elasticsearch/script/expression/ExpressionScript.java index ef3376c4e7239..e5746c78fb5e0 100644 --- a/src/main/java/org/elasticsearch/script/expression/ExpressionScript.java +++ b/src/main/java/org/elasticsearch/script/expression/ExpressionScript.java @@ -120,7 +120,14 @@ public void setNextDocId(int d) { } @Override - public void setNextScore(float score) { scorer.score = score; } + public void setNextScore(float score) { + // TODO: fix this API to remove setNextScore and just use a Scorer + // Expressions know if they use the score or not, and should be able to pull from the scorer only + // if they need it. Right now, score can only be used within a ScriptScoreFunction. But there shouldn't + // be any reason a script values or aggregation can't use the score. It is also possible + // these layers are preventing inlining of scoring into expressions. + scorer.score = score; + } @Override public void setNextReader(AtomicReaderContext leaf) { From caa1d60d66524871844c8cf6720c23f59bc5320c Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Fri, 11 Jul 2014 11:17:21 -0700 Subject: [PATCH 11/16] Fix a test bug, changed execution exception to compilation exception in most cases, and added docs for expression scripts. --- docs/reference/modules/scripting.asciidoc | 35 ++++++++++++++--- .../script/expression/ExpressionScript.java | 2 +- .../ExpressionScriptCompilationException.java | 3 ++ .../ExpressionScriptEngineService.java | 14 +++---- .../search/lookup/DocLookup.java | 6 ++- .../ExpressionScriptComparisonBenchmark.java | 30 ++++++++++++++ .../expression/ExpressionScriptTests.java | 39 ++++++++++--------- 7 files changed, 96 insertions(+), 33 deletions(-) create mode 100644 src/test/java/org/elasticsearch/benchmark/scripts/ExpressionScriptComparisonBenchmark.java diff --git a/docs/reference/modules/scripting.asciidoc b/docs/reference/modules/scripting.asciidoc index 7106bc4153e16..44920d4474c57 100644 --- a/docs/reference/modules/scripting.asciidoc +++ b/docs/reference/modules/scripting.asciidoc @@ -17,8 +17,8 @@ different languages. Currently supported plugins are `lang-javascript` for JavaScript, `lang-mvel` for Mvel, and `lang-python` for Python. All places where a `script` parameter can be used, a `lang` parameter (on the same level) can be provided to define the language of the -script. The `lang` options are `groovy`, `js`, `mvel`, `python`, and -`native`. +script. The `lang` options are `groovy`, `js`, `mvel`, `python`, +`expression` and `native`. added[1.2.0, Dynamic scripting is disabled for non-sandboxed languages by default since version 1.2.0] @@ -184,14 +184,38 @@ the name of the script as the `script`. Note, the scripts need to be in the classpath of elasticsearch. One simple way to do it is to create a directory under plugins (choose a -descriptive name), and place the jar / classes files there, they will be +descriptive name), and place the jar / classes files there. They will be automatically loaded. +[float] +=== Lucene Expressions Scripts + +Lucene's expressions module provides a mechanism to compile a +`javascript` expression to bytecode. This allows very fast execution, +as if you had written a `native` script. Expression scripts can be +used in `script_score`, `script_fields` and numeric aggregation scripts. + +See the link:http://lucene.apache.org/core/4_9_0/expressions/index.html?org/apache/lucene/expressions/package-summary.html[expressions module documentation] +for details on what operators and functions are available. + +Variables in `expression` scripts are available to access: + +* Single valued document fields, e.g. `doc['myfield'].value` +* Parameters passed into the script, e.g. `mymodifier` +* The current document's score, `_score` (only available when used in a `script_score`) + +There are a few limitations relative to other script languages: + +* Only numeric fields may be accessed +* Stored fields are not available +* If a field is sparse (only some documents contain a value), documents missing the field will have a value of `0` + [float] === Score -In all scripts that can be used in facets, allow to access the current -doc score using `doc.score`. +In all scripts that can be used in facets, the current +document's score is accessible in `doc.score`. When using a `script_score`, +the current score is available in `_score`. [float] === Computing scores based on terms in scripts @@ -402,3 +426,4 @@ integer with the value of `8`, the result is `0` even though you were expecting it to be `0.125`. You may need to enforce precision by explicitly using a double like `1.0/num` in order to get the expected result. + diff --git a/src/main/java/org/elasticsearch/script/expression/ExpressionScript.java b/src/main/java/org/elasticsearch/script/expression/ExpressionScript.java index e5746c78fb5e0..1d8bf8b3d67c4 100644 --- a/src/main/java/org/elasticsearch/script/expression/ExpressionScript.java +++ b/src/main/java/org/elasticsearch/script/expression/ExpressionScript.java @@ -134,7 +134,7 @@ public void setNextReader(AtomicReaderContext leaf) { try { values = source.getValues(context, leaf); } catch (IOException e) { - throw new ExpressionScriptExecutionException("Failed to run expression", e); + throw new ExpressionScriptExecutionException("Expression failed to bind for segment", e); } } diff --git a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptCompilationException.java b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptCompilationException.java index 48a3df1be65f6..e02401efc1535 100644 --- a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptCompilationException.java +++ b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptCompilationException.java @@ -30,4 +30,7 @@ public class ExpressionScriptCompilationException extends ElasticsearchException public ExpressionScriptCompilationException(String msg, ParseException e) { super(msg, e); } + public ExpressionScriptCompilationException(String msg) { + super(msg); + } } diff --git a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java index f8633c70ebd0a..1b4608462f798 100644 --- a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java +++ b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java @@ -105,31 +105,31 @@ public SearchScript search(Object compiledScript, SearchLookup lookup, @Nullable if (value instanceof Number) { bindings.add(variable, new DoubleConstValueSource(((Number)value).doubleValue())); } else { - throw new ExpressionScriptExecutionException("Parameter [" + variable + "] must be a numeric type"); + throw new ExpressionScriptCompilationException("Parameter [" + variable + "] must be a numeric type"); } } else { XVariableContext[] parts = XVariableContext.parse(variable); if (parts[0].text.equals("doc") == false) { - throw new ExpressionScriptExecutionException("Unknown variable [" + parts[0].text + "] in expression"); + throw new ExpressionScriptCompilationException("Unknown variable [" + parts[0].text + "] in expression"); } if (parts.length < 2 || parts[1].type != XVariableContext.Type.STR_INDEX) { - throw new ExpressionScriptExecutionException("Variable 'doc' in expression must be used with a specific field like: doc['myfield'].value"); + throw new ExpressionScriptCompilationException("Variable 'doc' in expression must be used with a specific field like: doc['myfield'].value"); } if (parts.length < 3 || parts[2].type != XVariableContext.Type.MEMBER || parts[2].text.equals("value") == false) { - throw new ExpressionScriptExecutionException("Invalid member for field data in expression. Only '.value' is currently supported."); + throw new ExpressionScriptCompilationException("Invalid member for field data in expression. Only '.value' is currently supported."); } String fieldname = parts[1].text; FieldMapper field = mapper.smartNameFieldMapper(fieldname); if (field == null) { - throw new ExpressionScriptExecutionException("Field [" + fieldname + "] used in expression does not exist in mappings"); + throw new ExpressionScriptCompilationException("Field [" + fieldname + "] used in expression does not exist in mappings"); } if (field.isNumeric() == false) { // TODO: more context (which expression?) - throw new ExpressionScriptExecutionException("Field [" + fieldname + "] used in expression must be numeric"); + throw new ExpressionScriptCompilationException("Field [" + fieldname + "] used in expression must be numeric"); } - IndexFieldData fieldData = lookup.doc().fieldDataService.getForField((NumberFieldMapper)field); + IndexFieldData fieldData = lookup.doc().fieldDataService().getForField((NumberFieldMapper)field); bindings.add(variable, new FieldDataValueSource(fieldData)); } } diff --git a/src/main/java/org/elasticsearch/search/lookup/DocLookup.java b/src/main/java/org/elasticsearch/search/lookup/DocLookup.java index 6970cdf744f3f..9cf56e66da9b8 100644 --- a/src/main/java/org/elasticsearch/search/lookup/DocLookup.java +++ b/src/main/java/org/elasticsearch/search/lookup/DocLookup.java @@ -42,7 +42,7 @@ public class DocLookup implements Map { private final Map localCacheFieldData = Maps.newHashMapWithExpectedSize(4); private final MapperService mapperService; - public final IndexFieldDataService fieldDataService; + private final IndexFieldDataService fieldDataService; @Nullable private final String[] types; @@ -63,6 +63,10 @@ public MapperService mapperService() { return this.mapperService; } + public IndexFieldDataService fieldDataService() { + return this.fieldDataService; + } + public void setNextReader(AtomicReaderContext context) { if (this.reader == context) { // if we are called with the same reader, don't invalidate source return; diff --git a/src/test/java/org/elasticsearch/benchmark/scripts/ExpressionScriptComparisonBenchmark.java b/src/test/java/org/elasticsearch/benchmark/scripts/ExpressionScriptComparisonBenchmark.java new file mode 100644 index 0000000000000..6306e0fab8c07 --- /dev/null +++ b/src/test/java/org/elasticsearch/benchmark/scripts/ExpressionScriptComparisonBenchmark.java @@ -0,0 +1,30 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.benchmark.scripts; + +import java.util.Random; + +public class ExpressionScriptComparisonBenchmark { + + public static void main(String[] args) { + Random r = new Random(); + + } +} diff --git a/src/test/java/org/elasticsearch/script/expression/ExpressionScriptTests.java b/src/test/java/org/elasticsearch/script/expression/ExpressionScriptTests.java index 1540f5457a46c..775406dac2fd9 100644 --- a/src/test/java/org/elasticsearch/script/expression/ExpressionScriptTests.java +++ b/src/test/java/org/elasticsearch/script/expression/ExpressionScriptTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.ShardSearchFailure; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.aggregations.metrics.stats.Stats; import org.elasticsearch.test.ElasticsearchIntegrationTest; @@ -107,8 +108,8 @@ public void testMissingField() throws Exception { runScript("doc['bogus'].value"); fail("Expected missing field to cause failure"); } catch (SearchPhaseExecutionException e) { - assertThat(ExceptionsHelper.detailedMessage(e) + "should have contained ExpressionScriptExecutionException", - ExceptionsHelper.detailedMessage(e).contains("ExpressionScriptExecutionException"), equalTo(true)); + assertThat(ExceptionsHelper.detailedMessage(e) + "should have contained ExpressionScriptCompilationException", + ExceptionsHelper.detailedMessage(e).contains("ExpressionScriptCompilationException"), equalTo(true)); assertThat(ExceptionsHelper.detailedMessage(e) + "should have contained missing field error", ExceptionsHelper.detailedMessage(e).contains("does not exist in mappings"), equalTo(true)); } @@ -149,8 +150,8 @@ public void testNonNumericParam() { runScript("a", "a", "astring"); fail("Expected string parameter to cause failure"); } catch (SearchPhaseExecutionException e) { - assertThat(ExceptionsHelper.detailedMessage(e) + "should have contained ExpressionScriptExecutiontException", - ExceptionsHelper.detailedMessage(e).contains("ExpressionScriptExecutionException"), equalTo(true)); + assertThat(ExceptionsHelper.detailedMessage(e) + "should have contained ExpressionScriptCompilationException", + ExceptionsHelper.detailedMessage(e).contains("ExpressionScriptCompilationException"), equalTo(true)); assertThat(ExceptionsHelper.detailedMessage(e) + "should have contained non-numeric parameter error", ExceptionsHelper.detailedMessage(e).contains("must be a numeric type"), equalTo(true)); } @@ -162,8 +163,8 @@ public void testNonNumericField() { runScript("doc['text'].value"); fail("Expected text field to cause execution failure"); } catch (SearchPhaseExecutionException e) { - assertThat(ExceptionsHelper.detailedMessage(e) + "should have contained ExpressionScriptExecutiontException", - ExceptionsHelper.detailedMessage(e).contains("ExpressionScriptExecutionException"), equalTo(true)); + assertThat(ExceptionsHelper.detailedMessage(e) + "should have contained ExpressionScriptCompilationException", + ExceptionsHelper.detailedMessage(e).contains("ExpressionScriptCompilationException"), equalTo(true)); assertThat(ExceptionsHelper.detailedMessage(e) + "should have contained non-numeric field error", ExceptionsHelper.detailedMessage(e).contains("must be numeric"), equalTo(true)); } @@ -175,8 +176,8 @@ public void testInvalidGlobalVariable() { runScript("bogus"); fail("Expected bogus variable to cause execution failure"); } catch (SearchPhaseExecutionException e) { - assertThat(ExceptionsHelper.detailedMessage(e) + "should have contained ExpressionScriptExecutiontException", - ExceptionsHelper.detailedMessage(e).contains("ExpressionScriptExecutionException"), equalTo(true)); + assertThat(ExceptionsHelper.detailedMessage(e) + "should have contained ExpressionScriptCompilationException", + ExceptionsHelper.detailedMessage(e).contains("ExpressionScriptCompilationException"), equalTo(true)); assertThat(ExceptionsHelper.detailedMessage(e) + "should have contained unknown variable error", ExceptionsHelper.detailedMessage(e).contains("Unknown variable"), equalTo(true)); } @@ -188,8 +189,8 @@ public void testDocWithoutField() { runScript("doc"); fail("Expected doc variable without field to cause execution failure"); } catch (SearchPhaseExecutionException e) { - assertThat(ExceptionsHelper.detailedMessage(e) + "should have contained ExpressionScriptExecutiontException", - ExceptionsHelper.detailedMessage(e).contains("ExpressionScriptExecutionException"), equalTo(true)); + assertThat(ExceptionsHelper.detailedMessage(e) + "should have contained ExpressionScriptCompilationException", + ExceptionsHelper.detailedMessage(e).contains("ExpressionScriptCompilationException"), equalTo(true)); assertThat(ExceptionsHelper.detailedMessage(e) + "should have contained a missing specific field error", ExceptionsHelper.detailedMessage(e).contains("must be used with a specific field"), equalTo(true)); } @@ -201,8 +202,8 @@ public void testInvalidFieldMember() { runScript("doc['foo'].bogus"); fail("Expected bogus field member to cause execution failure"); } catch (SearchPhaseExecutionException e) { - assertThat(ExceptionsHelper.detailedMessage(e) + "should have contained ExpressionScriptExecutiontException", - ExceptionsHelper.detailedMessage(e).contains("ExpressionScriptExecutionException"), equalTo(true)); + assertThat(ExceptionsHelper.detailedMessage(e) + "should have contained ExpressionScriptCompilationException", + ExceptionsHelper.detailedMessage(e).contains("ExpressionScriptCompilationException"), equalTo(true)); assertThat(ExceptionsHelper.detailedMessage(e) + "should have contained field member error", ExceptionsHelper.detailedMessage(e).contains("Invalid member for field"), equalTo(true)); } @@ -252,19 +253,19 @@ public void testStringSpecialValueVariable() throws Exception { "lang: \"expression\"" + "}}}}"; - Throwable t; + String message; try { // shards that don't have docs with the "text" field will not fail, // so we may or may not get a total failure SearchResponse rsp = client().prepareSearch("test").setSource(req).get(); assertThat(rsp.getShardFailures().length, greaterThan(0)); // at least the shards containing the docs should have failed - t = rsp.getShardFailures()[0].failure(); + message = rsp.getShardFailures()[0].reason(); } catch (SearchPhaseExecutionException e) { - t = e; + message = ExceptionsHelper.detailedMessage(e); } - assertThat(ExceptionsHelper.detailedMessage(t) + "should have contained ExpressionScriptExecutiontException", - ExceptionsHelper.detailedMessage(t).contains("ExpressionScriptExecutionException"), equalTo(true)); - assertThat(ExceptionsHelper.detailedMessage(t) + "should have contained text variable error", - ExceptionsHelper.detailedMessage(t).contains("text variable"), equalTo(true)); + assertThat(message + "should have contained ExpressionScriptExecutionException", + message.contains("ExpressionScriptExecutionException"), equalTo(true)); + assertThat(message + "should have contained text variable error", + message.contains("text variable"), equalTo(true)); } } From 17091e293140dc2449976519cf51a9f13ab45fdc Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Fri, 11 Jul 2014 11:21:29 -0700 Subject: [PATCH 12/16] Remove empty benchmark file --- .../ExpressionScriptComparisonBenchmark.java | 30 ------------------- 1 file changed, 30 deletions(-) delete mode 100644 src/test/java/org/elasticsearch/benchmark/scripts/ExpressionScriptComparisonBenchmark.java diff --git a/src/test/java/org/elasticsearch/benchmark/scripts/ExpressionScriptComparisonBenchmark.java b/src/test/java/org/elasticsearch/benchmark/scripts/ExpressionScriptComparisonBenchmark.java deleted file mode 100644 index 6306e0fab8c07..0000000000000 --- a/src/test/java/org/elasticsearch/benchmark/scripts/ExpressionScriptComparisonBenchmark.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.benchmark.scripts; - -import java.util.Random; - -public class ExpressionScriptComparisonBenchmark { - - public static void main(String[] args) { - Random r = new Random(); - - } -} From 93f8bb3a376e5682ed58513cd46b8bd3fa04d559 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Fri, 11 Jul 2014 11:23:09 -0700 Subject: [PATCH 13/16] Change expression disk extension to match langauge name (to require less documentation). --- .../script/expression/ExpressionScriptEngineService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java index 1b4608462f798..5ab643768c129 100644 --- a/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java +++ b/src/main/java/org/elasticsearch/script/expression/ExpressionScriptEngineService.java @@ -59,7 +59,7 @@ public String[] types() { @Override public String[] extensions() { - return new String[]{"expr"}; + return new String[]{"expression"}; } @Override From ec49e968f87bfa3f6f48d0b663f06c4bd2e150a2 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Fri, 11 Jul 2014 17:19:26 -0700 Subject: [PATCH 14/16] Tweaked documentation and add benchmark comparing expressions to native and groovy scripts --- docs/reference/modules/scripting.asciidoc | 4 +- .../scripts/expression/NativeScript1.java | 45 +++++ .../scripts/expression/NativeScript2.java | 45 +++++ .../scripts/expression/NativeScript3.java | 45 +++++ .../scripts/expression/NativeScript4.java | 45 +++++ .../expression/NativeScriptPlugin.java | 43 +++++ .../expression/ScriptComparisonBenchmark.java | 164 ++++++++++++++++++ 7 files changed, 389 insertions(+), 2 deletions(-) create mode 100644 src/test/java/org/elasticsearch/benchmark/scripts/expression/NativeScript1.java create mode 100644 src/test/java/org/elasticsearch/benchmark/scripts/expression/NativeScript2.java create mode 100644 src/test/java/org/elasticsearch/benchmark/scripts/expression/NativeScript3.java create mode 100644 src/test/java/org/elasticsearch/benchmark/scripts/expression/NativeScript4.java create mode 100644 src/test/java/org/elasticsearch/benchmark/scripts/expression/NativeScriptPlugin.java create mode 100644 src/test/java/org/elasticsearch/benchmark/scripts/expression/ScriptComparisonBenchmark.java diff --git a/docs/reference/modules/scripting.asciidoc b/docs/reference/modules/scripting.asciidoc index 44920d4474c57..182f4a63dca32 100644 --- a/docs/reference/modules/scripting.asciidoc +++ b/docs/reference/modules/scripting.asciidoc @@ -193,9 +193,9 @@ automatically loaded. Lucene's expressions module provides a mechanism to compile a `javascript` expression to bytecode. This allows very fast execution, as if you had written a `native` script. Expression scripts can be -used in `script_score`, `script_fields` and numeric aggregation scripts. +used in `script_score`, `script_fields`, sort scripts and numeric aggregation scripts. -See the link:http://lucene.apache.org/core/4_9_0/expressions/index.html?org/apache/lucene/expressions/package-summary.html[expressions module documentation] +See the link:http://lucene.apache.org/core/4_9_0/expressions/index.html?org/apache/lucene/expressions/js/package-summary.html[expressions module documentation] for details on what operators and functions are available. Variables in `expression` scripts are available to access: diff --git a/src/test/java/org/elasticsearch/benchmark/scripts/expression/NativeScript1.java b/src/test/java/org/elasticsearch/benchmark/scripts/expression/NativeScript1.java new file mode 100644 index 0000000000000..d4b7a0d6716c0 --- /dev/null +++ b/src/test/java/org/elasticsearch/benchmark/scripts/expression/NativeScript1.java @@ -0,0 +1,45 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.benchmark.scripts.expression; + +import org.elasticsearch.common.Nullable; +import org.elasticsearch.script.AbstractSearchScript; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.NativeScriptFactory; + +import java.util.Map; + +public class NativeScript1 extends AbstractSearchScript { + + public static class Factory implements NativeScriptFactory { + + @Override + public ExecutableScript newScript(@Nullable Map params) { + return new NativeScript1(); + } + } + + public static final String NATIVE_SCRIPT_1 = "native_1"; + + @Override + public Object run() { + return docFieldLongs("x").getValue(); + } +} diff --git a/src/test/java/org/elasticsearch/benchmark/scripts/expression/NativeScript2.java b/src/test/java/org/elasticsearch/benchmark/scripts/expression/NativeScript2.java new file mode 100644 index 0000000000000..acb374bf68ee4 --- /dev/null +++ b/src/test/java/org/elasticsearch/benchmark/scripts/expression/NativeScript2.java @@ -0,0 +1,45 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.benchmark.scripts.expression; + +import org.elasticsearch.common.Nullable; +import org.elasticsearch.script.AbstractSearchScript; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.NativeScriptFactory; + +import java.util.Map; + +public class NativeScript2 extends AbstractSearchScript { + + public static class Factory implements NativeScriptFactory { + + @Override + public ExecutableScript newScript(@Nullable Map params) { + return new NativeScript2(); + } + } + + public static final String NATIVE_SCRIPT_2 = "native_2"; + + @Override + public Object run() { + return docFieldLongs("x").getValue() + docFieldDoubles("y").getValue(); + } +} diff --git a/src/test/java/org/elasticsearch/benchmark/scripts/expression/NativeScript3.java b/src/test/java/org/elasticsearch/benchmark/scripts/expression/NativeScript3.java new file mode 100644 index 0000000000000..b57cde7cac2aa --- /dev/null +++ b/src/test/java/org/elasticsearch/benchmark/scripts/expression/NativeScript3.java @@ -0,0 +1,45 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.benchmark.scripts.expression; + +import org.elasticsearch.common.Nullable; +import org.elasticsearch.script.AbstractSearchScript; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.NativeScriptFactory; + +import java.util.Map; + +public class NativeScript3 extends AbstractSearchScript { + + public static class Factory implements NativeScriptFactory { + + @Override + public ExecutableScript newScript(@Nullable Map params) { + return new NativeScript3(); + } + } + + public static final String NATIVE_SCRIPT_3 = "native_3"; + + @Override + public Object run() { + return 1.2 * docFieldLongs("x").getValue() / docFieldDoubles("y").getValue(); + } +} diff --git a/src/test/java/org/elasticsearch/benchmark/scripts/expression/NativeScript4.java b/src/test/java/org/elasticsearch/benchmark/scripts/expression/NativeScript4.java new file mode 100644 index 0000000000000..d87d1deeaab73 --- /dev/null +++ b/src/test/java/org/elasticsearch/benchmark/scripts/expression/NativeScript4.java @@ -0,0 +1,45 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.benchmark.scripts.expression; + +import org.elasticsearch.common.Nullable; +import org.elasticsearch.script.AbstractSearchScript; +import org.elasticsearch.script.ExecutableScript; +import org.elasticsearch.script.NativeScriptFactory; + +import java.util.Map; + +public class NativeScript4 extends AbstractSearchScript { + + public static class Factory implements NativeScriptFactory { + + @Override + public ExecutableScript newScript(@Nullable Map params) { + return new NativeScript4(); + } + } + + public static final String NATIVE_SCRIPT_4 = "native_4"; + + @Override + public Object run() { + return Math.sqrt(Math.abs(docFieldDoubles("z").getValue())) + Math.log(Math.abs(docFieldLongs("x").getValue() * docFieldDoubles("y").getValue())); + } +} diff --git a/src/test/java/org/elasticsearch/benchmark/scripts/expression/NativeScriptPlugin.java b/src/test/java/org/elasticsearch/benchmark/scripts/expression/NativeScriptPlugin.java new file mode 100644 index 0000000000000..c2e8bb9ff7d6d --- /dev/null +++ b/src/test/java/org/elasticsearch/benchmark/scripts/expression/NativeScriptPlugin.java @@ -0,0 +1,43 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.benchmark.scripts.expression; + +import org.elasticsearch.plugins.AbstractPlugin; +import org.elasticsearch.script.ScriptModule; + +public class NativeScriptPlugin extends AbstractPlugin { + + @Override + public String name() { + return "native-benchmark-scripts"; + } + + @Override + public String description() { + return "Native benchmark script"; + } + + public void onModule(ScriptModule module) { + module.registerScript(NativeScript1.NATIVE_SCRIPT_1, NativeScript1.Factory.class); + module.registerScript(NativeScript2.NATIVE_SCRIPT_2, NativeScript2.Factory.class); + module.registerScript(NativeScript3.NATIVE_SCRIPT_3, NativeScript3.Factory.class); + module.registerScript(NativeScript4.NATIVE_SCRIPT_4, NativeScript4.Factory.class); + } +} diff --git a/src/test/java/org/elasticsearch/benchmark/scripts/expression/ScriptComparisonBenchmark.java b/src/test/java/org/elasticsearch/benchmark/scripts/expression/ScriptComparisonBenchmark.java new file mode 100644 index 0000000000000..f573376d9d191 --- /dev/null +++ b/src/test/java/org/elasticsearch/benchmark/scripts/expression/ScriptComparisonBenchmark.java @@ -0,0 +1,164 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.benchmark.scripts.expression; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.bulk.BulkRequestBuilder; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.IndicesAdminClient; +import org.elasticsearch.common.StopWatch; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.node.Node; +import org.elasticsearch.search.sort.ScriptSortBuilder; +import org.elasticsearch.search.sort.SortBuilders; +import org.joda.time.PeriodType; + +import java.util.Random; + +import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder; +import static org.elasticsearch.node.NodeBuilder.nodeBuilder; + +public class ScriptComparisonBenchmark { + + static final String clusterName = ScriptComparisonBenchmark.class.getSimpleName(); + static final String indexName = "test"; + + static String[] langs = { + "expression", + "native", + "groovy" + }; + static String[][] scripts = { + // the first value is the "reference" version (pure math) + { + "x", + "doc['x'].value", + NativeScript1.NATIVE_SCRIPT_1, + "doc['x'].value" + }, { + "x + y", + "doc['x'].value + doc['y'].value", + NativeScript2.NATIVE_SCRIPT_2, + "doc['x'].value + doc['y'].value", + }, { + "1.2 * x / y", + "1.2 * doc['x'].value / doc['y'].value", + NativeScript3.NATIVE_SCRIPT_3, + "1.2 * doc['x'].value / doc['y'].value", + }, { + "sqrt(abs(z)) + ln(abs(x * y))", + "sqrt(abs(doc['z'].value)) + ln(abs(doc['x'].value * doc['y'].value))", + NativeScript4.NATIVE_SCRIPT_4, + "sqrt(abs(doc['z'].value)) + log(abs(doc['x'].value * doc['y'].value))" + } + }; + + public static void main(String[] args) throws Exception { + int numDocs = 1000000; + int numQueries = 1000; + Client client = setupIndex(); + indexDocs(client, numDocs); + + for (int scriptNum = 0; scriptNum < scripts.length; ++scriptNum) { + runBenchmark(client, scriptNum, numQueries); + } + } + + static void runBenchmark(Client client, int scriptNum, int numQueries) { + System.out.println(""); + System.out.println("Script: " + scripts[scriptNum][0]); + System.out.println("--------------------------------"); + for (int langNum = 0; langNum < langs.length; ++langNum) { + String lang = langs[langNum]; + String script = scripts[scriptNum][langNum + 1]; + + timeQueries(client, lang, script, numQueries / 10); // warmup + TimeValue time = timeQueries(client, lang, script, numQueries); + printResults(lang, time, numQueries); + } + } + + static Client setupIndex() throws Exception { + // create cluster + Settings settings = settingsBuilder().put("plugin.types", NativeScriptPlugin.class.getName()) + .put("name", "node1") + .build(); + Node node1 = nodeBuilder().clusterName(clusterName).settings(settings).node(); + Client client = node1.client(); + client.admin().cluster().prepareHealth(indexName).setWaitForGreenStatus().setTimeout("10s").execute().actionGet(); + + // delete the index, if it exists + try { + client.admin().indices().prepareDelete(indexName).execute().actionGet(); + } catch (ElasticsearchException e) { + // ok if the index didn't exist + } + + // create mappings + IndicesAdminClient admin = client.admin().indices(); + admin.prepareCreate(indexName).addMapping("doc", "x", "type=long", "y", "type=double"); + + client.admin().cluster().prepareHealth(indexName).setWaitForGreenStatus().setTimeout("10s").execute().actionGet(); + return client; + } + + static void indexDocs(Client client, int numDocs) { + System.out.print("Indexing " + numDocs + " random docs..."); + BulkRequestBuilder bulkRequest = client.prepareBulk(); + Random r = new Random(1); + for (int i = 0; i < numDocs; i++) { + bulkRequest.add(client.prepareIndex("test", "doc", Integer.toString(i)) + .setSource("x", r.nextInt(), "y", r.nextDouble(), "z", r.nextDouble())); + + if (i % 1000 == 0) { + bulkRequest.execute().actionGet(); + bulkRequest = client.prepareBulk(); + } + } + bulkRequest.execute().actionGet(); + client.admin().indices().prepareRefresh("test").execute().actionGet(); + client.admin().indices().prepareFlush("test").setFull(true).execute().actionGet(); + System.out.println("done"); + } + + static TimeValue timeQueries(Client client, String lang, String script, int numQueries) { + ScriptSortBuilder sort = SortBuilders.scriptSort(script, "number").lang(lang); + SearchRequestBuilder req = client.prepareSearch(indexName) + .setQuery(QueryBuilders.matchAllQuery()) + .addSort(sort); + + StopWatch timer = new StopWatch(); + timer.start(); + for (int i = 0; i < numQueries; ++i) { + req.get(); + } + timer.stop(); + return timer.totalTime(); + } + + static void printResults(String lang, TimeValue time, int numQueries) { + long avgReq = time.millis() / numQueries; + System.out.println(lang + ": " + time.format(PeriodType.seconds()) + " (" + avgReq + " msec per req)"); + } + +} From 1a8154b8e58fa8b81a966ec6d5bfb779daf8c2ae Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Mon, 14 Jul 2014 07:20:17 -0700 Subject: [PATCH 15/16] Adding experimental warning to docs --- docs/reference/modules/scripting.asciidoc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/reference/modules/scripting.asciidoc b/docs/reference/modules/scripting.asciidoc index 182f4a63dca32..5ec55fa707050 100644 --- a/docs/reference/modules/scripting.asciidoc +++ b/docs/reference/modules/scripting.asciidoc @@ -190,6 +190,11 @@ automatically loaded. [float] === Lucene Expressions Scripts +[WARNING] +======================== +This feature is *experimental* and subject to change in future versions. +======================== + Lucene's expressions module provides a mechanism to compile a `javascript` expression to bytecode. This allows very fast execution, as if you had written a `native` script. Expression scripts can be From 140f7da966264e1817ec0f091ac14d3be7f45a96 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Mon, 14 Jul 2014 14:09:36 -0700 Subject: [PATCH 16/16] Address more review comments --- pom.xml | 2 +- .../script/expression/ExpressionScript.java | 12 +-- .../expression/ExpressionScriptTests.java | 97 ++++++++++--------- 3 files changed, 56 insertions(+), 55 deletions(-) diff --git a/pom.xml b/pom.xml index aa1b49b27b06b..04c2c7e8ce2bd 100644 --- a/pom.xml +++ b/pom.xml @@ -880,7 +880,7 @@ ${project.build.directory}/lib - lucene*, log4j*, jna*, spatial4j*, jts*, groovy*, + lucene*, log4j*, jna*, spatial4j*, jts*, groovy* directory perm diff --git a/src/main/java/org/elasticsearch/script/expression/ExpressionScript.java b/src/main/java/org/elasticsearch/script/expression/ExpressionScript.java index 1d8bf8b3d67c4..a1e2913ccc249 100644 --- a/src/main/java/org/elasticsearch/script/expression/ExpressionScript.java +++ b/src/main/java/org/elasticsearch/script/expression/ExpressionScript.java @@ -78,13 +78,13 @@ public long cost() { } } - Expression expression; - XSimpleBindings bindings; - CannedScorer scorer; - ValueSource source; + final Expression expression; + final XSimpleBindings bindings; + final CannedScorer scorer; + final ValueSource source; + final Map context; + final ReplaceableConstValueSource specialValue; // _value FunctionValues values; - Map context; - ReplaceableConstValueSource specialValue; // _value ExpressionScript(Expression e, XSimpleBindings b, ReplaceableConstValueSource v) { expression = e; diff --git a/src/test/java/org/elasticsearch/script/expression/ExpressionScriptTests.java b/src/test/java/org/elasticsearch/script/expression/ExpressionScriptTests.java index 775406dac2fd9..fc64dda79e0dc 100644 --- a/src/test/java/org/elasticsearch/script/expression/ExpressionScriptTests.java +++ b/src/test/java/org/elasticsearch/script/expression/ExpressionScriptTests.java @@ -20,14 +20,30 @@ package org.elasticsearch.script.expression; import org.elasticsearch.ExceptionsHelper; +import org.elasticsearch.action.get.GetRequestBuilder; import org.elasticsearch.action.search.SearchPhaseExecutionException; +import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder; +import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.metrics.stats.Stats; +import org.elasticsearch.search.sort.SortBuilder; +import org.elasticsearch.search.sort.SortBuilders; +import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.ElasticsearchIntegrationTest; import org.elasticsearch.test.hamcrest.ElasticsearchAssertions; +import java.io.IOException; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; @@ -35,25 +51,18 @@ public class ExpressionScriptTests extends ElasticsearchIntegrationTest { private SearchResponse runScript(String script, Object... params) { ensureGreen("test"); - StringBuilder buf = new StringBuilder(); - buf.append("{query: {match_all:{}}, sort: [{\"_uid\": {\"order\":\"asc\"}}], script_fields: {foo: {lang: \"expression\", script: \"" + script + "\""); - if (params.length > 0) { - assert(params.length % 2 == 0); - buf.append(",params:{"); - for (int i = 0; i < params.length; i += 2) { - if (i != 0) buf.append(","); - buf.append("\"" + params[i] + "\":"); - Object v = params[i + 1]; - if (v instanceof String) { - buf.append("\"" + v + "\""); - } else { - buf.append(v); - } - } - buf.append("}"); + + Map paramsMap = new HashMap<>(); + assert(params.length % 2 == 0); + for (int i = 0; i < params.length; i += 2) { + paramsMap.put(params[i].toString(), params[i + 1]); } - buf.append("}}}"); - return client().prepareSearch("test").setSource(buf.toString()).get(); + + SearchRequestBuilder req = new SearchRequestBuilder(client()).setIndices("test"); + req.setQuery(QueryBuilders.matchAllQuery()) + .addSort(SortBuilders.fieldSort("_uid") + .order(SortOrder.ASC)).addScriptField("foo", "expression", script, paramsMap); + return req.get(); } public void testBasic() throws Exception { @@ -69,16 +78,13 @@ public void testScore() throws Exception { createIndex("test"); ensureGreen("test"); indexRandom(true, - client().prepareIndex("test", "doc", "1").setSource("text", "hello goodbye"), - client().prepareIndex("test", "doc", "2").setSource("text", "hello hello hello goodbye"), - client().prepareIndex("test", "doc", "3").setSource("text", "hello hello goodebye")); - String req = "{query: {function_score: {query:{term:{text:\"hello\"}}," + - "boost_mode: \"replace\"," + - "script_score: {" + - "script: \"1 / _score\"," + // invert the order given by score - "lang: \"expression\"" + - "}}}}"; - SearchResponse rsp = client().prepareSearch("test").setSource(req).get(); + client().prepareIndex("test", "doc", "1").setSource("text", "hello goodbye"), + client().prepareIndex("test", "doc", "2").setSource("text", "hello hello hello goodbye"), + client().prepareIndex("test", "doc", "3").setSource("text", "hello hello goodebye")); + ScoreFunctionBuilder score = ScoreFunctionBuilders.scriptFunction("1 / _score", "expression", Collections.EMPTY_MAP); + SearchRequestBuilder req = new SearchRequestBuilder(client()).setIndices("test"); + req.setQuery(QueryBuilders.functionScoreQuery(QueryBuilders.termQuery("text", "hello"), score).boostMode("replace")); + SearchResponse rsp = req.get(); SearchHits hits = rsp.getHits(); assertEquals(3, hits.getTotalHits()); assertEquals("1", hits.getAt(0).getId()); @@ -214,20 +220,16 @@ public void testSpecialValueVariable() throws Exception { createIndex("test"); ensureGreen("test"); indexRandom(true, - client().prepareIndex("test", "doc", "1").setSource("x", 5, "y", 1.2), - client().prepareIndex("test", "doc", "2").setSource("x", 10, "y", 1.4), - client().prepareIndex("test", "doc", "3").setSource("x", 13, "y", 1.8)); - String req = "{query: {match_all:{}}, aggs: {" + - "int_agg: {stats: {" + - "field: \"x\"," + - "script: \"_value * 3\"," + - "lang: \"expression\"" + - "}},double_agg: {stats: {" + - "field: \"y\"," + - "script: \"_value - 1.1\"," + - "lang: \"expression\"" + - "}}}}"; - SearchResponse rsp = client().prepareSearch("test").setSource(req).get(); + client().prepareIndex("test", "doc", "1").setSource("x", 5, "y", 1.2), + client().prepareIndex("test", "doc", "2").setSource("x", 10, "y", 1.4), + client().prepareIndex("test", "doc", "3").setSource("x", 13, "y", 1.8)); + + SearchRequestBuilder req = new SearchRequestBuilder(client()).setIndices("test"); + req.setQuery(QueryBuilders.matchAllQuery()) + .addAggregation(AggregationBuilders.stats("int_agg").field("x").script("_value * 3").lang("expression")) + .addAggregation(AggregationBuilders.stats("double_agg").field("y").script("_value - 1.1").lang("expression")); + + SearchResponse rsp = req.get(); assertEquals(3, rsp.getHits().getTotalHits()); Stats stats = rsp.getAggregations().get("int_agg"); @@ -247,17 +249,16 @@ public void testStringSpecialValueVariable() throws Exception { client().prepareIndex("test", "doc", "1").setSource("text", "hello"), client().prepareIndex("test", "doc", "2").setSource("text", "goodbye"), client().prepareIndex("test", "doc", "3").setSource("text", "hello")); - String req = "{query: {match_all:{}}, aggs: {term_agg: {terms: {" + - "field: \"text\"," + - "script: \"_value\"," + - "lang: \"expression\"" + - "}}}}"; + + SearchRequestBuilder req = new SearchRequestBuilder(client()).setIndices("test"); + req.setQuery(QueryBuilders.matchAllQuery()) + .addAggregation(AggregationBuilders.terms("term_agg").field("text").script("_value").lang("expression")); String message; try { // shards that don't have docs with the "text" field will not fail, // so we may or may not get a total failure - SearchResponse rsp = client().prepareSearch("test").setSource(req).get(); + SearchResponse rsp = req.get(); assertThat(rsp.getShardFailures().length, greaterThan(0)); // at least the shards containing the docs should have failed message = rsp.getShardFailures()[0].reason(); } catch (SearchPhaseExecutionException e) {