From 4d663436b9a33983f179e77f7178c350985e3925 Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Sun, 8 Jul 2018 12:00:23 -0700 Subject: [PATCH 01/50] Painless: Restructure Definition/Whitelist (#31879) Create lookup package rename Definition to PainlessLookup and move to lookup package rename Definition.Method to PainlessMethod rename Definition.MethodKey to PainlessMethod rename Definition.Field to PainlessField rename Definition.Struct to PainlessClass rename Definition.Cast to PainlessCast rename Whitelist.Struct to WhitelistClass rename Whitelist.Constructor to WhitelistConstructor rename Whitelist.Method to WhitelistMethod rename Whitelist.Field to WhitelistField --- .../elasticsearch/painless/spi/Whitelist.java | 169 +------ .../painless/spi/WhitelistClass.java | 76 ++++ .../painless/spi/WhitelistConstructor.java | 48 ++ .../painless/spi/WhitelistField.java | 46 ++ .../painless/spi/WhitelistLoader.java | 66 +-- .../painless/spi/WhitelistMethod.java | 76 ++++ .../painless/AnalyzerCaster.java | 391 ++++++++-------- .../org/elasticsearch/painless/Compiler.java | 37 +- .../java/org/elasticsearch/painless/Def.java | 121 ++--- .../elasticsearch/painless/DefBootstrap.java | 65 +-- .../org/elasticsearch/painless/DefMath.java | 57 ++- .../elasticsearch/painless/FunctionRef.java | 39 +- .../org/elasticsearch/painless/Locals.java | 43 +- .../elasticsearch/painless/MethodWriter.java | 6 +- .../painless/PainlessExplainError.java | 6 +- .../painless/PainlessScriptEngine.java | 5 +- .../painless/ScriptClassInfo.java | 25 +- .../painless/WriterConstants.java | 15 +- .../painless/antlr/EnhancedPainlessLexer.java | 10 +- .../elasticsearch/painless/antlr/Walker.java | 14 +- .../painless/lookup/PainlessCast.java | 67 +++ .../painless/lookup/PainlessClass.java | 103 +++++ .../painless/lookup/PainlessField.java | 43 ++ .../PainlessLookup.java} | 420 +++--------------- .../painless/lookup/PainlessMethod.java | 130 ++++++ .../painless/lookup/PainlessMethodKey.java | 75 ++++ .../painless/node/AExpression.java | 8 +- .../painless/node/EAssignment.java | 8 +- .../elasticsearch/painless/node/EBinary.java | 26 +- .../painless/node/ECallLocal.java | 8 +- .../painless/node/ECapturingFunctionRef.java | 10 +- .../elasticsearch/painless/node/ECast.java | 10 +- .../elasticsearch/painless/node/EComp.java | 20 +- .../painless/node/EExplicit.java | 2 +- .../painless/node/EFunctionRef.java | 16 +- .../painless/node/EInstanceof.java | 8 +- .../elasticsearch/painless/node/ELambda.java | 22 +- .../painless/node/EListInit.java | 15 +- .../elasticsearch/painless/node/EMapInit.java | 15 +- .../painless/node/ENewArray.java | 2 +- .../elasticsearch/painless/node/ENewObj.java | 14 +- .../elasticsearch/painless/node/ENull.java | 4 +- .../elasticsearch/painless/node/EStatic.java | 2 +- .../elasticsearch/painless/node/EUnary.java | 12 +- .../elasticsearch/painless/node/PBrace.java | 10 +- .../painless/node/PCallInvoke.java | 18 +- .../elasticsearch/painless/node/PField.java | 31 +- .../painless/node/PSubCallInvoke.java | 6 +- .../painless/node/PSubDefArray.java | 2 +- .../painless/node/PSubDefCall.java | 2 +- .../painless/node/PSubDefField.java | 2 +- .../painless/node/PSubField.java | 10 +- .../painless/node/PSubListShortcut.java | 18 +- .../painless/node/PSubMapShortcut.java | 18 +- .../painless/node/PSubShortcut.java | 8 +- .../elasticsearch/painless/node/SCatch.java | 2 +- .../painless/node/SDeclaration.java | 2 +- .../elasticsearch/painless/node/SEach.java | 9 +- .../painless/node/SFunction.java | 18 +- .../elasticsearch/painless/node/SSource.java | 16 +- .../painless/node/SSubEachArray.java | 8 +- .../painless/node/SSubEachIterable.java | 21 +- .../painless/AnalyzerCasterTests.java | 4 +- .../painless/BaseClassTests.java | 47 +- .../elasticsearch/painless/DebugTests.java | 15 +- .../org/elasticsearch/painless/Debugger.java | 3 +- .../painless/DefBootstrapTests.java | 23 +- .../painless/PainlessDocGenerator.java | 75 ++-- .../painless/ScriptTestCase.java | 7 +- .../painless/node/NodeToStringTests.java | 46 +- 70 files changed, 1510 insertions(+), 1266 deletions(-) create mode 100644 modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistClass.java create mode 100644 modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistConstructor.java create mode 100644 modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistField.java create mode 100644 modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistMethod.java create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessCast.java create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClass.java create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessField.java rename modules/lang-painless/src/main/java/org/elasticsearch/painless/{Definition.java => lookup/PainlessLookup.java} (71%) create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethod.java create mode 100644 modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethodKey.java diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/Whitelist.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/Whitelist.java index e715eb0090c7f..55b64b0420df1 100644 --- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/Whitelist.java +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/Whitelist.java @@ -24,13 +24,14 @@ import java.util.Objects; /** - * Whitelist contains data structures designed to be used to generate a white-list of Java classes, + * Whitelist contains data structures designed to be used to generate a whitelist of Java classes, * constructors, methods, and fields that can be used within a Painless script at both compile-time * and run-time. * - * A white-list consists of several pieces with {@link Struct}s as the top level. Each {@link Struct} - * will contain zero-to-many {@link Constructor}s, {@link Method}s, and {@link Field}s which are what - * will be available with a Painless script. See each individual white-list object for more detail. + * A whitelist consists of several pieces with {@link WhitelistClass}s as the top level. Each + * {@link WhitelistClass} will contain zero-to-many {@link WhitelistConstructor}s, {@link WhitelistMethod}s, and + * {@link WhitelistField}s which are what will be available with a Painless script. See each individual + * whitelist object for more detail. */ public final class Whitelist { @@ -54,166 +55,14 @@ public final class Whitelist { public static final List BASE_WHITELISTS = Collections.singletonList(WhitelistLoader.loadFromResourceFiles(Whitelist.class, BASE_WHITELIST_FILES)); - /** - * Struct represents the equivalent of a Java class in Painless complete with super classes, - * constructors, methods, and fields. In Painless a class is known as a struct primarily to avoid - * naming conflicts internally. There must be a one-to-one mapping of struct names to Java classes. - * Though, since multiple white-lists may be combined into a single white-list for a specific - * {@link org.elasticsearch.script.ScriptContext}, as long as multiple structs representing the same - * Java class have the same Painless type name and have legal constructor/method overloading they - * can be merged together. - * - * Structs in Painless allow for arity overloading for constructors and methods. Arity overloading - * means that multiple constructors are allowed for a single struct as long as they have a different - * number of parameter types, and multiples methods with the same name are allowed for a single struct - * as long as they have the same return type and a different number of parameter types. - * - * Structs will automatically extend other white-listed structs if the Java class they represent is a - * subclass of other structs including Java interfaces. - */ - public static final class Struct { - - /** Information about where this struct was white-listed from. Can be used for error messages. */ - public final String origin; - - /** The Java class name this struct represents. */ - public final String javaClassName; - - /** - * Allow the Java class name to only be specified as the fully-qualified name. - */ - public final boolean onlyFQNJavaClassName; - - /** The {@link List} of white-listed ({@link Constructor}s) available to this struct. */ - public final List whitelistConstructors; - - /** The {@link List} of white-listed ({@link Method}s) available to this struct. */ - public final List whitelistMethods; - - /** The {@link List} of white-listed ({@link Field}s) available to this struct. */ - public final List whitelistFields; - - /** Standard constructor. All values must be not {@code null}. */ - public Struct(String origin, String javaClassName, boolean onlyFQNJavaClassName, - List whitelistConstructors, List whitelistMethods, List whitelistFields) { - this.origin = Objects.requireNonNull(origin); - this.javaClassName = Objects.requireNonNull(javaClassName); - this.onlyFQNJavaClassName = onlyFQNJavaClassName; - - this.whitelistConstructors = Collections.unmodifiableList(Objects.requireNonNull(whitelistConstructors)); - this.whitelistMethods = Collections.unmodifiableList(Objects.requireNonNull(whitelistMethods)); - this.whitelistFields = Collections.unmodifiableList(Objects.requireNonNull(whitelistFields)); - } - } - - /** - * Constructor represents the equivalent of a Java constructor available as a white-listed struct - * constructor within Painless. Constructors for Painless structs may be accessed exactly as - * constructors for Java classes are using the 'new' keyword. Painless structs may have multiple - * constructors as long as they comply with arity overloading described for {@link Struct}. - */ - public static final class Constructor { - - /** Information about where this constructor was white-listed from. Can be used for error messages. */ - public final String origin; - - /** - * A {@link List} of {@link String}s that are the Painless type names for the parameters of the - * constructor which can be used to look up the Java constructor through reflection. - */ - public final List painlessParameterTypeNames; - - /** Standard constructor. All values must be not {@code null}. */ - public Constructor(String origin, List painlessParameterTypeNames) { - this.origin = Objects.requireNonNull(origin); - this.painlessParameterTypeNames = Collections.unmodifiableList(Objects.requireNonNull(painlessParameterTypeNames)); - } - } - - /** - * Method represents the equivalent of a Java method available as a white-listed struct method - * within Painless. Methods for Painless structs may be accessed exactly as methods for Java classes - * are using the '.' operator on an existing struct variable/field. Painless structs may have multiple - * methods with the same name as long as they comply with arity overloading described for {@link Method}. - * - * Structs may also have additional methods that are not part of the Java class the struct represents - - * these are known as augmented methods. An augmented method can be added to a struct as a part of any - * Java class as long as the method is static and the first parameter of the method is the Java class - * represented by the struct. Note that the augmented method's parent Java class does not need to be - * white-listed. - */ - public static class Method { - - /** Information about where this method was white-listed from. Can be used for error messages. */ - public final String origin; - - /** - * The Java class name for the owner of an augmented method. If the method is not augmented - * this should be {@code null}. - */ - public final String javaAugmentedClassName; - - /** The Java method name used to look up the Java method through reflection. */ - public final String javaMethodName; - - /** - * The Painless type name for the return type of the method which can be used to look up the Java - * method through reflection. - */ - public final String painlessReturnTypeName; - - /** - * A {@link List} of {@link String}s that are the Painless type names for the parameters of the - * method which can be used to look up the Java method through reflection. - */ - public final List painlessParameterTypeNames; - - /** - * Standard constructor. All values must be not {@code null} with the exception of jAugmentedClass; - * jAugmentedClass will be {@code null} unless the method is augmented as described in the class documentation. - */ - public Method(String origin, String javaAugmentedClassName, String javaMethodName, - String painlessReturnTypeName, List painlessParameterTypeNames) { - this.origin = Objects.requireNonNull(origin); - this.javaAugmentedClassName = javaAugmentedClassName; - this.javaMethodName = javaMethodName; - this.painlessReturnTypeName = Objects.requireNonNull(painlessReturnTypeName); - this.painlessParameterTypeNames = Collections.unmodifiableList(Objects.requireNonNull(painlessParameterTypeNames)); - } - } - - /** - * Field represents the equivalent of a Java field available as a white-listed struct field - * within Painless. Fields for Painless structs may be accessed exactly as fields for Java classes - * are using the '.' operator on an existing struct variable/field. - */ - public static class Field { - - /** Information about where this method was white-listed from. Can be used for error messages. */ - public final String origin; - - /** The Java field name used to look up the Java field through reflection. */ - public final String javaFieldName; - - /** The Painless type name for the field which can be used to look up the Java field through reflection. */ - public final String painlessFieldTypeName; - - /** Standard constructor. All values must be not {@code null}. */ - public Field(String origin, String javaFieldName, String painlessFieldTypeName) { - this.origin = Objects.requireNonNull(origin); - this.javaFieldName = Objects.requireNonNull(javaFieldName); - this.painlessFieldTypeName = Objects.requireNonNull(painlessFieldTypeName); - } - } - - /** The {@link ClassLoader} used to look up the white-listed Java classes, constructors, methods, and fields. */ + /** The {@link ClassLoader} used to look up the whitelisted Java classes, constructors, methods, and fields. */ public final ClassLoader javaClassLoader; - /** The {@link List} of all the white-listed Painless structs. */ - public final List whitelistStructs; + /** The {@link List} of all the whitelisted Painless classes. */ + public final List whitelistStructs; /** Standard constructor. All values must be not {@code null}. */ - public Whitelist(ClassLoader javaClassLoader, List whitelistStructs) { + public Whitelist(ClassLoader javaClassLoader, List whitelistStructs) { this.javaClassLoader = Objects.requireNonNull(javaClassLoader); this.whitelistStructs = Collections.unmodifiableList(Objects.requireNonNull(whitelistStructs)); } diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistClass.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistClass.java new file mode 100644 index 0000000000000..12aa5f5bdd634 --- /dev/null +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistClass.java @@ -0,0 +1,76 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.painless.spi; + +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +/** + * Class represents the equivalent of a Java class in Painless complete with super classes, + * constructors, methods, and fields. There must be a one-to-one mapping of class names to Java + * classes. Though, since multiple whitelists may be combined into a single whitelist for a + * specific context, as long as multiple classes representing the same Java class have the same + * class name and have legal constructor/method overloading they can be merged together. + * + * Classes in Painless allow for arity overloading for constructors and methods. Arity overloading + * means that multiple constructors are allowed for a single class as long as they have a different + * number of parameters, and multiples methods with the same name are allowed for a single class + * as long as they have the same return type and a different number of parameters. + * + * Classes will automatically extend other whitelisted classes if the Java class they represent is a + * subclass of other classes including Java interfaces. + */ +public final class WhitelistClass { + + /** Information about where this class was white-listed from. Can be used for error messages. */ + public final String origin; + + /** The Java class name this class represents. */ + public final String javaClassName; + + /** + * Allow the Java class name to only be specified as the fully-qualified name. + */ + public final boolean onlyFQNJavaClassName; + + /** The {@link List} of whitelisted ({@link WhitelistConstructor}s) available to this class. */ + public final List whitelistConstructors; + + /** The {@link List} of whitelisted ({@link WhitelistMethod}s) available to this class. */ + public final List whitelistMethods; + + /** The {@link List} of whitelisted ({@link WhitelistField}s) available to this class. */ + public final List whitelistFields; + + /** Standard constructor. All values must be not {@code null}. */ + public WhitelistClass(String origin, String javaClassName, boolean onlyFQNJavaClassName, + List whitelistConstructors, + List whitelistMethods, + List whitelistFields) { + this.origin = Objects.requireNonNull(origin); + this.javaClassName = Objects.requireNonNull(javaClassName); + this.onlyFQNJavaClassName = onlyFQNJavaClassName; + + this.whitelistConstructors = Collections.unmodifiableList(Objects.requireNonNull(whitelistConstructors)); + this.whitelistMethods = Collections.unmodifiableList(Objects.requireNonNull(whitelistMethods)); + this.whitelistFields = Collections.unmodifiableList(Objects.requireNonNull(whitelistFields)); + } +} diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistConstructor.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistConstructor.java new file mode 100644 index 0000000000000..0e70552760208 --- /dev/null +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistConstructor.java @@ -0,0 +1,48 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.painless.spi; + +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +/** + * Constructor represents the equivalent of a Java constructor available as a whitelisted class + * constructor within Painless. Constructors for Painless classes may be accessed exactly as + * constructors for Java classes are using the 'new' keyword. Painless classes may have multiple + * constructors as long as they comply with arity overloading described for {@link WhitelistClass}. + */ +public final class WhitelistConstructor { + + /** Information about where this constructor was whitelisted from. Can be used for error messages. */ + public final String origin; + + /** + * A {@link List} of {@link String}s that are the Painless type names for the parameters of the + * constructor which can be used to look up the Java constructor through reflection. + */ + public final List painlessParameterTypeNames; + + /** Standard constructor. All values must be not {@code null}. */ + public WhitelistConstructor(String origin, List painlessParameterTypeNames) { + this.origin = Objects.requireNonNull(origin); + this.painlessParameterTypeNames = Collections.unmodifiableList(Objects.requireNonNull(painlessParameterTypeNames)); + } +} diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistField.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistField.java new file mode 100644 index 0000000000000..116aea98fcf89 --- /dev/null +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistField.java @@ -0,0 +1,46 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.painless.spi; + +import java.util.Objects; + +/** + * Field represents the equivalent of a Java field available as a whitelisted class field + * within Painless. Fields for Painless classes may be accessed exactly as fields for Java classes + * are using the '.' operator on an existing class variable/field. + */ +public class WhitelistField { + + /** Information about where this method was whitelisted from. Can be used for error messages. */ + public final String origin; + + /** The Java field name used to look up the Java field through reflection. */ + public final String javaFieldName; + + /** The Painless type name for the field which can be used to look up the Java field through reflection. */ + public final String painlessFieldTypeName; + + /** Standard constructor. All values must be not {@code null}. */ + public WhitelistField(String origin, String javaFieldName, String painlessFieldTypeName) { + this.origin = Objects.requireNonNull(origin); + this.javaFieldName = Objects.requireNonNull(javaFieldName); + this.painlessFieldTypeName = Objects.requireNonNull(painlessFieldTypeName); + } +} diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistLoader.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistLoader.java index 8817bfa274c60..b104d03f1ea82 100644 --- a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistLoader.java +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistLoader.java @@ -39,25 +39,25 @@ public final class WhitelistLoader { * {@link String}s with a single {@link Class} to be be used to load the resources where each {@link String} * is the path of a single text file. The {@link Class}'s {@link ClassLoader} will be used to lookup the Java * reflection objects for each individual {@link Class}, {@link Constructor}, {@link Method}, and {@link Field} - * specified as part of the white-list in the text file. + * specified as part of the whitelist in the text file. * - * A single pass is made through each file to collect all the information about each struct, constructor, method, - * and field. Most validation will be done at a later point after all white-lists have been gathered and their + * A single pass is made through each file to collect all the information about each class, constructor, method, + * and field. Most validation will be done at a later point after all whitelists have been gathered and their * merging takes place. * * A painless type name is one of the following: *
    *
  • def - The Painless dynamic type which is automatically included without a need to be - * white-listed.
    • - *
  • fully-qualified Java type name - Any white-listed Java class will have the equivalent name as + * whitelisted.
    • + *
  • fully-qualified Java type name - Any whitelisted Java class will have the equivalent name as * a Painless type name with the exception that any dollar symbols used as part of inner classes will * be replaced with dot symbols.
    • *
  • short Java type name - The text after the final dot symbol of any specified Java class. A - * short type Java name may be excluded by using the 'only_fqn' token during Painless struct parsing + * short type Java name may be excluded by using the 'only_fqn' token during Painless class parsing * as described later.
    • *
* - * The following can be parsed from each white-list text file: + * The following can be parsed from each whitelist text file: *
    *
  • Blank lines will be ignored by the parser.
    • *
  • Comments may be created starting with a pound '#' symbol and end with a newline. These will @@ -71,19 +71,19 @@ public final class WhitelistLoader { *
      *
    • A constructor may be specified starting with an opening parenthesis, followed by a * comma-delimited list of Painless type names corresponding to the type/class names for - * the equivalent Java parameter types (these must be white-listed as well), a closing + * the equivalent Java parameter types (these must be whitelisted as well), a closing * parenthesis, and a newline.
      • *
    • A method may be specified starting with a Painless type name for the return type, * followed by the Java name of the method (which will also be the Painless name for the * method), an opening parenthesis, a comma-delimited list of Painless type names * corresponding to the type/class names for the equivalent Java parameter types - * (these must be white-listed as well), a closing parenthesis, and a newline.
      • + * (these must be whitelisted as well), a closing parenthesis, and a newline. *
    • An augmented method may be specified starting with a Painless type name for the return * type, followed by the fully qualified Java name of the class the augmented method is - * part of (this class does not need to be white-listed), the Java name of the method + * part of (this class does not need to be whitelisted), the Java name of the method * (which will also be the Painless name for the method), an opening parenthesis, a * comma-delimited list of Painless type names corresponding to the type/class names - * for the equivalent Java parameter types (these must be white-listed as well), a closing + * for the equivalent Java parameter types (these must be whitelisted as well), a closing * parenthesis, and a newline.
      • *
    • A field may be specified starting with a Painless type name for the equivalent Java type * of the field, followed by the Java name of the field (which all be the Painless name @@ -99,7 +99,7 @@ public final class WhitelistLoader { * fully-qualified Java class name. Method argument types, method return types, and field types * must be specified with Painless type names (def, fully-qualified, or short) as described earlier. * - * The following example is used to create a single white-list text file: + * The following example is used to create a single whitelist text file: * * {@code * # primitive types @@ -132,10 +132,10 @@ public final class WhitelistLoader { * } */ public static Whitelist loadFromResourceFiles(Class resource, String... filepaths) { - List whitelistStructs = new ArrayList<>(); + List whitelistStructs = new ArrayList<>(); - // Execute a single pass through the white-list text files. This will gather all the - // constructors, methods, augmented methods, and fields for each white-listed struct. + // Execute a single pass through the whitelist text files. This will gather all the + // constructors, methods, augmented methods, and fields for each whitelisted class. for (String filepath : filepaths) { String line; int number = -1; @@ -146,9 +146,9 @@ public static Whitelist loadFromResourceFiles(Class resource, String... filep String whitelistStructOrigin = null; String javaClassName = null; boolean onlyFQNJavaClassName = false; - List whitelistConstructors = null; - List whitelistMethods = null; - List whitelistFields = null; + List whitelistConstructors = null; + List whitelistMethods = null; + List whitelistFields = null; while ((line = reader.readLine()) != null) { number = reader.getLineNumber(); @@ -159,13 +159,13 @@ public static Whitelist loadFromResourceFiles(Class resource, String... filep continue; } - // Handle a new struct by resetting all the variables necessary to construct a new Whitelist.Struct for the white-list. + // Handle a new class by resetting all the variables necessary to construct a new WhitelistClass for the whitelist. // Expects the following format: 'class' ID 'only_fqn'? '{' '\n' if (line.startsWith("class ")) { // Ensure the final token of the line is '{'. if (line.endsWith("{") == false) { throw new IllegalArgumentException( - "invalid struct definition: failed to parse class opening bracket [" + line + "]"); + "invalid class definition: failed to parse class opening bracket [" + line + "]"); } // Parse the Java class name. @@ -175,29 +175,29 @@ public static Whitelist loadFromResourceFiles(Class resource, String... filep if (tokens.length == 2 && "only_fqn".equals(tokens[1])) { onlyFQNJavaClassName = true; } else if (tokens.length != 1) { - throw new IllegalArgumentException("invalid struct definition: failed to parse class name [" + line + "]"); + throw new IllegalArgumentException("invalid class definition: failed to parse class name [" + line + "]"); } whitelistStructOrigin = "[" + filepath + "]:[" + number + "]"; javaClassName = tokens[0]; - // Reset all the constructors, methods, and fields to support a new struct. + // Reset all the constructors, methods, and fields to support a new class. whitelistConstructors = new ArrayList<>(); whitelistMethods = new ArrayList<>(); whitelistFields = new ArrayList<>(); - // Handle the end of a struct, by creating a new Whitelist.Struct with all the previously gathered - // constructors, methods, augmented methods, and fields, and adding it to the list of white-listed structs. + // Handle the end of a class, by creating a new WhitelistClass with all the previously gathered + // constructors, methods, augmented methods, and fields, and adding it to the list of whitelisted classes. // Expects the following format: '}' '\n' } else if (line.equals("}")) { if (javaClassName == null) { - throw new IllegalArgumentException("invalid struct definition: extraneous closing bracket"); + throw new IllegalArgumentException("invalid class definition: extraneous closing bracket"); } - whitelistStructs.add(new Whitelist.Struct(whitelistStructOrigin, javaClassName, onlyFQNJavaClassName, + whitelistStructs.add(new WhitelistClass(whitelistStructOrigin, javaClassName, onlyFQNJavaClassName, whitelistConstructors, whitelistMethods, whitelistFields)); - // Set all the variables to null to ensure a new struct definition is found before other parsable values. + // Set all the variables to null to ensure a new class definition is found before other parsable values. whitelistStructOrigin = null; javaClassName = null; onlyFQNJavaClassName = false; @@ -210,7 +210,7 @@ public static Whitelist loadFromResourceFiles(Class resource, String... filep // Mark the origin of this parsable object. String origin = "[" + filepath + "]:[" + number + "]"; - // Ensure we have a defined struct before adding any constructors, methods, augmented methods, or fields. + // Ensure we have a defined class before adding any constructors, methods, augmented methods, or fields. if (javaClassName == null) { throw new IllegalArgumentException("invalid object definition: expected a class name [" + line + "]"); } @@ -232,7 +232,7 @@ public static Whitelist loadFromResourceFiles(Class resource, String... filep tokens = new String[0]; } - whitelistConstructors.add(new Whitelist.Constructor(origin, Arrays.asList(tokens))); + whitelistConstructors.add(new WhitelistConstructor(origin, Arrays.asList(tokens))); // Handle the case for a method or augmented method definition. // Expects the following format: ID ID? ID '(' ( ID ( ',' ID )* )? ')' '\n' @@ -271,7 +271,7 @@ public static Whitelist loadFromResourceFiles(Class resource, String... filep tokens = new String[0]; } - whitelistMethods.add(new Whitelist.Method(origin, javaAugmentedClassName, javaMethodName, + whitelistMethods.add(new WhitelistMethod(origin, javaAugmentedClassName, javaMethodName, painlessReturnTypeName, Arrays.asList(tokens))); // Handle the case for a field definition. @@ -285,14 +285,14 @@ public static Whitelist loadFromResourceFiles(Class resource, String... filep throw new IllegalArgumentException("invalid field definition: unexpected format [" + line + "]"); } - whitelistFields.add(new Whitelist.Field(origin, tokens[1], tokens[0])); + whitelistFields.add(new WhitelistField(origin, tokens[1], tokens[0])); } } } - // Ensure all structs end with a '}' token before the end of the file. + // Ensure all classes end with a '}' token before the end of the file. if (javaClassName != null) { - throw new IllegalArgumentException("invalid struct definition: expected closing bracket"); + throw new IllegalArgumentException("invalid class definition: expected closing bracket"); } } catch (Exception exception) { throw new RuntimeException("error in [" + filepath + "] at line [" + number + "]", exception); diff --git a/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistMethod.java b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistMethod.java new file mode 100644 index 0000000000000..df86619055b08 --- /dev/null +++ b/modules/lang-painless/spi/src/main/java/org/elasticsearch/painless/spi/WhitelistMethod.java @@ -0,0 +1,76 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.painless.spi; + +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +/** + * Method represents the equivalent of a Java method available as a whitelisted class method + * within Painless. Methods for Painless classes may be accessed exactly as methods for Java classes + * are using the '.' operator on an existing class variable/field. Painless classes may have multiple + * methods with the same name as long as they comply with arity overloading described for {@link WhitelistMethod}. + * + * Classes may also have additional methods that are not part of the Java class the class represents - + * these are known as augmented methods. An augmented method can be added to a class as a part of any + * Java class as long as the method is static and the first parameter of the method is the Java class + * represented by the class. Note that the augmented method's parent Java class does not need to be + * whitelisted. + */ +public class WhitelistMethod { + + /** Information about where this method was whitelisted from. Can be used for error messages. */ + public final String origin; + + /** + * The Java class name for the owner of an augmented method. If the method is not augmented + * this should be {@code null}. + */ + public final String javaAugmentedClassName; + + /** The Java method name used to look up the Java method through reflection. */ + public final String javaMethodName; + + /** + * The Painless type name for the return type of the method which can be used to look up the Java + * method through reflection. + */ + public final String painlessReturnTypeName; + + /** + * A {@link List} of {@link String}s that are the Painless type names for the parameters of the + * method which can be used to look up the Java method through reflection. + */ + public final List painlessParameterTypeNames; + + /** + * Standard constructor. All values must be not {@code null} with the exception of jAugmentedClass; + * jAugmentedClass will be {@code null} unless the method is augmented as described in the class documentation. + */ + public WhitelistMethod(String origin, String javaAugmentedClassName, String javaMethodName, + String painlessReturnTypeName, List painlessParameterTypeNames) { + this.origin = Objects.requireNonNull(origin); + this.javaAugmentedClassName = javaAugmentedClassName; + this.javaMethodName = javaMethodName; + this.painlessReturnTypeName = Objects.requireNonNull(painlessReturnTypeName); + this.painlessParameterTypeNames = Collections.unmodifiableList(Objects.requireNonNull(painlessParameterTypeNames)); + } +} diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java index abba62de39c19..69ef57faad63e 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/AnalyzerCaster.java @@ -19,8 +19,9 @@ package org.elasticsearch.painless; -import org.elasticsearch.painless.Definition.Cast; -import org.elasticsearch.painless.Definition.def; +import org.elasticsearch.painless.lookup.PainlessLookup; +import org.elasticsearch.painless.lookup.PainlessCast; +import org.elasticsearch.painless.lookup.PainlessLookup.def; import java.util.Objects; @@ -30,7 +31,7 @@ */ public final class AnalyzerCaster { - public static Cast getLegalCast(Location location, Class actual, Class expected, boolean explicit, boolean internal) { + public static PainlessCast getLegalCast(Location location, Class actual, Class expected, boolean explicit, boolean internal) { Objects.requireNonNull(actual); Objects.requireNonNull(expected); @@ -40,421 +41,421 @@ public static Cast getLegalCast(Location location, Class actual, Class exp if (actual == def.class) { if (expected == boolean.class) { - return Cast.unboxTo(def.class, Boolean.class, explicit, boolean.class); + return PainlessCast.unboxTo(def.class, Boolean.class, explicit, boolean.class); } else if (expected == byte.class) { - return Cast.unboxTo(def.class, Byte.class, explicit, byte.class); + return PainlessCast.unboxTo(def.class, Byte.class, explicit, byte.class); } else if (expected == short.class) { - return Cast.unboxTo(def.class, Short.class, explicit, short.class); + return PainlessCast.unboxTo(def.class, Short.class, explicit, short.class); } else if (expected == char.class) { - return Cast.unboxTo(def.class, Character.class, explicit, char.class); + return PainlessCast.unboxTo(def.class, Character.class, explicit, char.class); } else if (expected == int.class) { - return Cast.unboxTo(def.class, Integer.class, explicit, int.class); + return PainlessCast.unboxTo(def.class, Integer.class, explicit, int.class); } else if (expected == long.class) { - return Cast.unboxTo(def.class, Long.class, explicit, long.class); + return PainlessCast.unboxTo(def.class, Long.class, explicit, long.class); } else if (expected == float.class) { - return Cast.unboxTo(def.class, Float.class, explicit, float.class); + return PainlessCast.unboxTo(def.class, Float.class, explicit, float.class); } else if (expected == double.class) { - return Cast.unboxTo(def.class, Double.class, explicit, double.class); + return PainlessCast.unboxTo(def.class, Double.class, explicit, double.class); } } else if (actual == Object.class) { if (expected == byte.class && explicit && internal) { - return Cast.unboxTo(Object.class, Byte.class, true, byte.class); + return PainlessCast.unboxTo(Object.class, Byte.class, true, byte.class); } else if (expected == short.class && explicit && internal) { - return Cast.unboxTo(Object.class, Short.class, true, short.class); + return PainlessCast.unboxTo(Object.class, Short.class, true, short.class); } else if (expected == char.class && explicit && internal) { - return Cast.unboxTo(Object.class, Character.class, true, char.class); + return PainlessCast.unboxTo(Object.class, Character.class, true, char.class); } else if (expected == int.class && explicit && internal) { - return Cast.unboxTo(Object.class, Integer.class, true, int.class); + return PainlessCast.unboxTo(Object.class, Integer.class, true, int.class); } else if (expected == long.class && explicit && internal) { - return Cast.unboxTo(Object.class, Long.class, true, long.class); + return PainlessCast.unboxTo(Object.class, Long.class, true, long.class); } else if (expected == float.class && explicit && internal) { - return Cast.unboxTo(Object.class, Float.class, true, float.class); + return PainlessCast.unboxTo(Object.class, Float.class, true, float.class); } else if (expected == double.class && explicit && internal) { - return Cast.unboxTo(Object.class, Double.class, true, double.class); + return PainlessCast.unboxTo(Object.class, Double.class, true, double.class); } } else if (actual == Number.class) { if (expected == byte.class && explicit && internal) { - return Cast.unboxTo(Number.class, Byte.class, true, byte.class); + return PainlessCast.unboxTo(Number.class, Byte.class, true, byte.class); } else if (expected == short.class && explicit && internal) { - return Cast.unboxTo(Number.class, Short.class, true, short.class); + return PainlessCast.unboxTo(Number.class, Short.class, true, short.class); } else if (expected == char.class && explicit && internal) { - return Cast.unboxTo(Number.class, Character.class, true, char.class); + return PainlessCast.unboxTo(Number.class, Character.class, true, char.class); } else if (expected == int.class && explicit && internal) { - return Cast.unboxTo(Number.class, Integer.class, true, int.class); + return PainlessCast.unboxTo(Number.class, Integer.class, true, int.class); } else if (expected == long.class && explicit && internal) { - return Cast.unboxTo(Number.class, Long.class, true, long.class); + return PainlessCast.unboxTo(Number.class, Long.class, true, long.class); } else if (expected == float.class && explicit && internal) { - return Cast.unboxTo(Number.class, Float.class, true, float.class); + return PainlessCast.unboxTo(Number.class, Float.class, true, float.class); } else if (expected == double.class && explicit && internal) { - return Cast.unboxTo(Number.class, Double.class, true, double.class); + return PainlessCast.unboxTo(Number.class, Double.class, true, double.class); } } else if (actual == String.class) { if (expected == char.class && explicit) { - return Cast.standard(String.class, char.class, true); + return PainlessCast.standard(String.class, char.class, true); } } else if (actual == boolean.class) { if (expected == def.class) { - return Cast.boxFrom(Boolean.class, def.class, explicit, boolean.class); + return PainlessCast.boxFrom(Boolean.class, def.class, explicit, boolean.class); } else if (expected == Object.class && internal) { - return Cast.boxFrom(Boolean.class, Object.class, explicit, boolean.class); + return PainlessCast.boxFrom(Boolean.class, Object.class, explicit, boolean.class); } else if (expected == Boolean.class && internal) { - return Cast.boxTo(boolean.class, boolean.class, explicit, boolean.class); + return PainlessCast.boxTo(boolean.class, boolean.class, explicit, boolean.class); } } else if (actual == byte.class) { if (expected == def.class) { - return Cast.boxFrom(Byte.class, def.class, explicit, byte.class); + return PainlessCast.boxFrom(Byte.class, def.class, explicit, byte.class); } else if (expected == Object.class && internal) { - return Cast.boxFrom(Byte.class, Object.class, explicit, byte.class); + return PainlessCast.boxFrom(Byte.class, Object.class, explicit, byte.class); } else if (expected == Number.class && internal) { - return Cast.boxFrom(Byte.class, Number.class, explicit, byte.class); + return PainlessCast.boxFrom(Byte.class, Number.class, explicit, byte.class); } else if (expected == short.class) { - return Cast.standard(byte.class, short.class, explicit); + return PainlessCast.standard(byte.class, short.class, explicit); } else if (expected == char.class && explicit) { - return Cast.standard(byte.class, char.class, true); + return PainlessCast.standard(byte.class, char.class, true); } else if (expected == int.class) { - return Cast.standard(byte.class, int.class, explicit); + return PainlessCast.standard(byte.class, int.class, explicit); } else if (expected == long.class) { - return Cast.standard(byte.class, long.class, explicit); + return PainlessCast.standard(byte.class, long.class, explicit); } else if (expected == float.class) { - return Cast.standard(byte.class, float.class, explicit); + return PainlessCast.standard(byte.class, float.class, explicit); } else if (expected == double.class) { - return Cast.standard(byte.class, double.class, explicit); + return PainlessCast.standard(byte.class, double.class, explicit); } else if (expected == Byte.class && internal) { - return Cast.boxTo(byte.class, byte.class, explicit, byte.class); + return PainlessCast.boxTo(byte.class, byte.class, explicit, byte.class); } else if (expected == Short.class && internal) { - return Cast.boxTo(byte.class, short.class, explicit, short.class); + return PainlessCast.boxTo(byte.class, short.class, explicit, short.class); } else if (expected == Character.class && explicit && internal) { - return Cast.boxTo(byte.class, char.class, true, char.class); + return PainlessCast.boxTo(byte.class, char.class, true, char.class); } else if (expected == Integer.class && internal) { - return Cast.boxTo(byte.class, int.class, explicit, int.class); + return PainlessCast.boxTo(byte.class, int.class, explicit, int.class); } else if (expected == Long.class && internal) { - return Cast.boxTo(byte.class, long.class, explicit, long.class); + return PainlessCast.boxTo(byte.class, long.class, explicit, long.class); } else if (expected == Float.class && internal) { - return Cast.boxTo(byte.class, float.class, explicit, float.class); + return PainlessCast.boxTo(byte.class, float.class, explicit, float.class); } else if (expected == Double.class && internal) { - return Cast.boxTo(byte.class, double.class, explicit, double.class); + return PainlessCast.boxTo(byte.class, double.class, explicit, double.class); } } else if (actual == short.class) { if (expected == def.class) { - return Cast.boxFrom(Short.class, def.class, explicit, short.class); + return PainlessCast.boxFrom(Short.class, def.class, explicit, short.class); } else if (expected == Object.class && internal) { - return Cast.boxFrom(Short.class, Object.class, explicit, short.class); + return PainlessCast.boxFrom(Short.class, Object.class, explicit, short.class); } else if (expected == Number.class && internal) { - return Cast.boxFrom(Short.class, Number.class, explicit, short.class); + return PainlessCast.boxFrom(Short.class, Number.class, explicit, short.class); } else if (expected == byte.class && explicit) { - return Cast.standard(short.class, byte.class, true); + return PainlessCast.standard(short.class, byte.class, true); } else if (expected == char.class && explicit) { - return Cast.standard(short.class, char.class, true); + return PainlessCast.standard(short.class, char.class, true); } else if (expected == int.class) { - return Cast.standard(short.class, int.class, explicit); + return PainlessCast.standard(short.class, int.class, explicit); } else if (expected == long.class) { - return Cast.standard(short.class, long.class, explicit); + return PainlessCast.standard(short.class, long.class, explicit); } else if (expected == float.class) { - return Cast.standard(short.class, float.class, explicit); + return PainlessCast.standard(short.class, float.class, explicit); } else if (expected == double.class) { - return Cast.standard(short.class, double.class, explicit); + return PainlessCast.standard(short.class, double.class, explicit); } else if (expected == Byte.class && explicit && internal) { - return Cast.boxTo(short.class, byte.class, true, byte.class); + return PainlessCast.boxTo(short.class, byte.class, true, byte.class); } else if (expected == Short.class && internal) { - return Cast.boxTo(short.class, short.class, explicit, short.class); + return PainlessCast.boxTo(short.class, short.class, explicit, short.class); } else if (expected == Character.class && explicit && internal) { - return Cast.boxTo(short.class, char.class, true, char.class); + return PainlessCast.boxTo(short.class, char.class, true, char.class); } else if (expected == Integer.class && internal) { - return Cast.boxTo(short.class, int.class, explicit, int.class); + return PainlessCast.boxTo(short.class, int.class, explicit, int.class); } else if (expected == Long.class && internal) { - return Cast.boxTo(short.class, long.class, explicit, long.class); + return PainlessCast.boxTo(short.class, long.class, explicit, long.class); } else if (expected == Float.class && internal) { - return Cast.boxTo(short.class, float.class, explicit, float.class); + return PainlessCast.boxTo(short.class, float.class, explicit, float.class); } else if (expected == Double.class && internal) { - return Cast.boxTo(short.class, double.class, explicit, double.class); + return PainlessCast.boxTo(short.class, double.class, explicit, double.class); } } else if (actual == char.class) { if (expected == def.class) { - return Cast.boxFrom(Character.class, def.class, explicit, char.class); + return PainlessCast.boxFrom(Character.class, def.class, explicit, char.class); } else if (expected == Object.class && internal) { - return Cast.boxFrom(Character.class, Object.class, explicit, char.class); + return PainlessCast.boxFrom(Character.class, Object.class, explicit, char.class); } else if (expected == Number.class && internal) { - return Cast.boxFrom(Character.class, Number.class, explicit, char.class); + return PainlessCast.boxFrom(Character.class, Number.class, explicit, char.class); } else if (expected == String.class) { - return Cast.standard(char.class, String.class, explicit); + return PainlessCast.standard(char.class, String.class, explicit); } else if (expected == byte.class && explicit) { - return Cast.standard(char.class, byte.class, true); + return PainlessCast.standard(char.class, byte.class, true); } else if (expected == short.class && explicit) { - return Cast.standard(char.class, short.class, true); + return PainlessCast.standard(char.class, short.class, true); } else if (expected == int.class) { - return Cast.standard(char.class, int.class, explicit); + return PainlessCast.standard(char.class, int.class, explicit); } else if (expected == long.class) { - return Cast.standard(char.class, long.class, explicit); + return PainlessCast.standard(char.class, long.class, explicit); } else if (expected == float.class) { - return Cast.standard(char.class, float.class, explicit); + return PainlessCast.standard(char.class, float.class, explicit); } else if (expected == double.class) { - return Cast.standard(char.class, double.class, explicit); + return PainlessCast.standard(char.class, double.class, explicit); } else if (expected == Byte.class && explicit && internal) { - return Cast.boxTo(char.class, byte.class, true, byte.class); + return PainlessCast.boxTo(char.class, byte.class, true, byte.class); } else if (expected == Short.class && internal) { - return Cast.boxTo(char.class, short.class, explicit, short.class); + return PainlessCast.boxTo(char.class, short.class, explicit, short.class); } else if (expected == Character.class && internal) { - return Cast.boxTo(char.class, char.class, true, char.class); + return PainlessCast.boxTo(char.class, char.class, true, char.class); } else if (expected == Integer.class && internal) { - return Cast.boxTo(char.class, int.class, explicit, int.class); + return PainlessCast.boxTo(char.class, int.class, explicit, int.class); } else if (expected == Long.class && internal) { - return Cast.boxTo(char.class, long.class, explicit, long.class); + return PainlessCast.boxTo(char.class, long.class, explicit, long.class); } else if (expected == Float.class && internal) { - return Cast.boxTo(char.class, float.class, explicit, float.class); + return PainlessCast.boxTo(char.class, float.class, explicit, float.class); } else if (expected == Double.class && internal) { - return Cast.boxTo(char.class, double.class, explicit, double.class); + return PainlessCast.boxTo(char.class, double.class, explicit, double.class); } } else if (actual == int.class) { if (expected == def.class) { - return Cast.boxFrom(Integer.class, def.class, explicit, int.class); + return PainlessCast.boxFrom(Integer.class, def.class, explicit, int.class); } else if (expected == Object.class && internal) { - return Cast.boxFrom(Integer.class, Object.class, explicit, int.class); + return PainlessCast.boxFrom(Integer.class, Object.class, explicit, int.class); } else if (expected == Number.class && internal) { - return Cast.boxFrom(Integer.class, Number.class, explicit, int.class); + return PainlessCast.boxFrom(Integer.class, Number.class, explicit, int.class); } else if (expected == byte.class && explicit) { - return Cast.standard(int.class, byte.class, true); + return PainlessCast.standard(int.class, byte.class, true); } else if (expected == char.class && explicit) { - return Cast.standard(int.class, char.class, true); + return PainlessCast.standard(int.class, char.class, true); } else if (expected == short.class && explicit) { - return Cast.standard(int.class, short.class, true); + return PainlessCast.standard(int.class, short.class, true); } else if (expected == long.class) { - return Cast.standard(int.class, long.class, explicit); + return PainlessCast.standard(int.class, long.class, explicit); } else if (expected == float.class) { - return Cast.standard(int.class, float.class, explicit); + return PainlessCast.standard(int.class, float.class, explicit); } else if (expected == double.class) { - return Cast.standard(int.class, double.class, explicit); + return PainlessCast.standard(int.class, double.class, explicit); } else if (expected == Byte.class && explicit && internal) { - return Cast.boxTo(int.class, byte.class, true, byte.class); + return PainlessCast.boxTo(int.class, byte.class, true, byte.class); } else if (expected == Short.class && explicit && internal) { - return Cast.boxTo(int.class, short.class, true, short.class); + return PainlessCast.boxTo(int.class, short.class, true, short.class); } else if (expected == Character.class && explicit && internal) { - return Cast.boxTo(int.class, char.class, true, char.class); + return PainlessCast.boxTo(int.class, char.class, true, char.class); } else if (expected == Integer.class && internal) { - return Cast.boxTo(int.class, int.class, explicit, int.class); + return PainlessCast.boxTo(int.class, int.class, explicit, int.class); } else if (expected == Long.class && internal) { - return Cast.boxTo(int.class, long.class, explicit, long.class); + return PainlessCast.boxTo(int.class, long.class, explicit, long.class); } else if (expected == Float.class && internal) { - return Cast.boxTo(int.class, float.class, explicit, float.class); + return PainlessCast.boxTo(int.class, float.class, explicit, float.class); } else if (expected == Double.class && internal) { - return Cast.boxTo(int.class, double.class, explicit, double.class); + return PainlessCast.boxTo(int.class, double.class, explicit, double.class); } } else if (actual == long.class) { if (expected == def.class) { - return Cast.boxFrom(Long.class, def.class, explicit, long.class); + return PainlessCast.boxFrom(Long.class, def.class, explicit, long.class); } else if (expected == Object.class && internal) { - return Cast.boxFrom(Long.class, Object.class, explicit, long.class); + return PainlessCast.boxFrom(Long.class, Object.class, explicit, long.class); } else if (expected == Number.class && internal) { - return Cast.boxFrom(Long.class, Number.class, explicit, long.class); + return PainlessCast.boxFrom(Long.class, Number.class, explicit, long.class); } else if (expected == byte.class && explicit) { - return Cast.standard(long.class, byte.class, true); + return PainlessCast.standard(long.class, byte.class, true); } else if (expected == char.class && explicit) { - return Cast.standard(long.class, char.class, true); + return PainlessCast.standard(long.class, char.class, true); } else if (expected == short.class && explicit) { - return Cast.standard(long.class, short.class, true); + return PainlessCast.standard(long.class, short.class, true); } else if (expected == int.class && explicit) { - return Cast.standard(long.class, int.class, true); + return PainlessCast.standard(long.class, int.class, true); } else if (expected == float.class) { - return Cast.standard(long.class, float.class, explicit); + return PainlessCast.standard(long.class, float.class, explicit); } else if (expected == double.class) { - return Cast.standard(long.class, double.class, explicit); + return PainlessCast.standard(long.class, double.class, explicit); } else if (expected == Byte.class && explicit && internal) { - return Cast.boxTo(long.class, byte.class, true, byte.class); + return PainlessCast.boxTo(long.class, byte.class, true, byte.class); } else if (expected == Short.class && explicit && internal) { - return Cast.boxTo(long.class, short.class, true, short.class); + return PainlessCast.boxTo(long.class, short.class, true, short.class); } else if (expected == Character.class && explicit && internal) { - return Cast.boxTo(long.class, char.class, true, char.class); + return PainlessCast.boxTo(long.class, char.class, true, char.class); } else if (expected == Integer.class && explicit && internal) { - return Cast.boxTo(long.class, int.class, true, int.class); + return PainlessCast.boxTo(long.class, int.class, true, int.class); } else if (expected == Long.class && internal) { - return Cast.boxTo(long.class, long.class, explicit, long.class); + return PainlessCast.boxTo(long.class, long.class, explicit, long.class); } else if (expected == Float.class && internal) { - return Cast.boxTo(long.class, float.class, explicit, float.class); + return PainlessCast.boxTo(long.class, float.class, explicit, float.class); } else if (expected == Double.class && internal) { - return Cast.boxTo(long.class, double.class, explicit, double.class); + return PainlessCast.boxTo(long.class, double.class, explicit, double.class); } } else if (actual == float.class) { if (expected == def.class) { - return Cast.boxFrom(Float.class, def.class, explicit, float.class); + return PainlessCast.boxFrom(Float.class, def.class, explicit, float.class); } else if (expected == Object.class && internal) { - return Cast.boxFrom(Float.class, Object.class, explicit, float.class); + return PainlessCast.boxFrom(Float.class, Object.class, explicit, float.class); } else if (expected == Number.class && internal) { - return Cast.boxFrom(Float.class, Number.class, explicit, float.class); + return PainlessCast.boxFrom(Float.class, Number.class, explicit, float.class); } else if (expected == byte.class && explicit) { - return Cast.standard(float.class, byte.class, true); + return PainlessCast.standard(float.class, byte.class, true); } else if (expected == char.class && explicit) { - return Cast.standard(float.class, char.class, true); + return PainlessCast.standard(float.class, char.class, true); } else if (expected == short.class && explicit) { - return Cast.standard(float.class, short.class, true); + return PainlessCast.standard(float.class, short.class, true); } else if (expected == int.class && explicit) { - return Cast.standard(float.class, int.class, true); + return PainlessCast.standard(float.class, int.class, true); } else if (expected == long.class && explicit) { - return Cast.standard(float.class, long.class, true); + return PainlessCast.standard(float.class, long.class, true); } else if (expected == double.class) { - return Cast.standard(float.class, double.class, explicit); + return PainlessCast.standard(float.class, double.class, explicit); } else if (expected == Byte.class && explicit && internal) { - return Cast.boxTo(float.class, byte.class, true, byte.class); + return PainlessCast.boxTo(float.class, byte.class, true, byte.class); } else if (expected == Short.class && explicit && internal) { - return Cast.boxTo(float.class, short.class, true, short.class); + return PainlessCast.boxTo(float.class, short.class, true, short.class); } else if (expected == Character.class && explicit && internal) { - return Cast.boxTo(float.class, char.class, true, char.class); + return PainlessCast.boxTo(float.class, char.class, true, char.class); } else if (expected == Integer.class && explicit && internal) { - return Cast.boxTo(float.class, int.class, true, int.class); + return PainlessCast.boxTo(float.class, int.class, true, int.class); } else if (expected == Long.class && explicit && internal) { - return Cast.boxTo(float.class, long.class, true, long.class); + return PainlessCast.boxTo(float.class, long.class, true, long.class); } else if (expected == Float.class && internal) { - return Cast.boxTo(float.class, float.class, explicit, float.class); + return PainlessCast.boxTo(float.class, float.class, explicit, float.class); } else if (expected == Double.class && internal) { - return Cast.boxTo(float.class, double.class, explicit, double.class); + return PainlessCast.boxTo(float.class, double.class, explicit, double.class); } } else if (actual == double.class) { if (expected == def.class) { - return Cast.boxFrom(Double.class, def.class, explicit, double.class); + return PainlessCast.boxFrom(Double.class, def.class, explicit, double.class); } else if (expected == Object.class && internal) { - return Cast.boxFrom(Double.class, Object.class, explicit, double.class); + return PainlessCast.boxFrom(Double.class, Object.class, explicit, double.class); } else if (expected == Number.class && internal) { - return Cast.boxFrom(Double.class, Number.class, explicit, double.class); + return PainlessCast.boxFrom(Double.class, Number.class, explicit, double.class); } else if (expected == byte.class && explicit) { - return Cast.standard(double.class, byte.class, true); + return PainlessCast.standard(double.class, byte.class, true); } else if (expected == char.class && explicit) { - return Cast.standard(double.class, char.class, true); + return PainlessCast.standard(double.class, char.class, true); } else if (expected == short.class && explicit) { - return Cast.standard(double.class, short.class, true); + return PainlessCast.standard(double.class, short.class, true); } else if (expected == int.class && explicit) { - return Cast.standard(double.class, int.class, true); + return PainlessCast.standard(double.class, int.class, true); } else if (expected == long.class && explicit) { - return Cast.standard(double.class, long.class, true); + return PainlessCast.standard(double.class, long.class, true); } else if (expected == float.class && explicit) { - return Cast.standard(double.class, float.class, true); + return PainlessCast.standard(double.class, float.class, true); } else if (expected == Byte.class && explicit && internal) { - return Cast.boxTo(double.class, byte.class, true, byte.class); + return PainlessCast.boxTo(double.class, byte.class, true, byte.class); } else if (expected == Short.class && explicit && internal) { - return Cast.boxTo(double.class, short.class, true, short.class); + return PainlessCast.boxTo(double.class, short.class, true, short.class); } else if (expected == Character.class && explicit && internal) { - return Cast.boxTo(double.class, char.class, true, char.class); + return PainlessCast.boxTo(double.class, char.class, true, char.class); } else if (expected == Integer.class && explicit && internal) { - return Cast.boxTo(double.class, int.class, true, int.class); + return PainlessCast.boxTo(double.class, int.class, true, int.class); } else if (expected == Long.class && explicit && internal) { - return Cast.boxTo(double.class, long.class, true, long.class); + return PainlessCast.boxTo(double.class, long.class, true, long.class); } else if (expected == Float.class && explicit && internal) { - return Cast.boxTo(double.class, float.class, true, float.class); + return PainlessCast.boxTo(double.class, float.class, true, float.class); } else if (expected == Double.class && internal) { - return Cast.boxTo(double.class, double.class, explicit, double.class); + return PainlessCast.boxTo(double.class, double.class, explicit, double.class); } } else if (actual == Boolean.class) { if (expected == boolean.class && internal) { - return Cast.unboxFrom(boolean.class, boolean.class, explicit, boolean.class); + return PainlessCast.unboxFrom(boolean.class, boolean.class, explicit, boolean.class); } } else if (actual == Byte.class) { if (expected == byte.class && internal) { - return Cast.unboxFrom(byte.class, byte.class, explicit, byte.class); + return PainlessCast.unboxFrom(byte.class, byte.class, explicit, byte.class); } else if (expected == short.class && internal) { - return Cast.unboxFrom(byte.class, short.class, explicit, byte.class); + return PainlessCast.unboxFrom(byte.class, short.class, explicit, byte.class); } else if (expected == char.class && explicit && internal) { - return Cast.unboxFrom(byte.class, char.class, true, byte.class); + return PainlessCast.unboxFrom(byte.class, char.class, true, byte.class); } else if (expected == int.class && internal) { - return Cast.unboxFrom(byte.class, int.class, explicit, byte.class); + return PainlessCast.unboxFrom(byte.class, int.class, explicit, byte.class); } else if (expected == long.class && internal) { - return Cast.unboxFrom(byte.class, long.class, explicit, byte.class); + return PainlessCast.unboxFrom(byte.class, long.class, explicit, byte.class); } else if (expected == float.class && internal) { - return Cast.unboxFrom(byte.class, float.class, explicit, byte.class); + return PainlessCast.unboxFrom(byte.class, float.class, explicit, byte.class); } else if (expected == double.class && internal) { - return Cast.unboxFrom(byte.class, double.class, explicit, byte.class); + return PainlessCast.unboxFrom(byte.class, double.class, explicit, byte.class); } } else if (actual == Short.class) { if (expected == byte.class && explicit && internal) { - return Cast.unboxFrom(short.class, byte.class, true, short.class); + return PainlessCast.unboxFrom(short.class, byte.class, true, short.class); } else if (expected == short.class && internal) { - return Cast.unboxFrom(short.class, short.class, explicit, short.class); + return PainlessCast.unboxFrom(short.class, short.class, explicit, short.class); } else if (expected == char.class && explicit && internal) { - return Cast.unboxFrom(short.class, char.class, true, short.class); + return PainlessCast.unboxFrom(short.class, char.class, true, short.class); } else if (expected == int.class && internal) { - return Cast.unboxFrom(short.class, int.class, explicit, short.class); + return PainlessCast.unboxFrom(short.class, int.class, explicit, short.class); } else if (expected == long.class && internal) { - return Cast.unboxFrom(short.class, long.class, explicit, short.class); + return PainlessCast.unboxFrom(short.class, long.class, explicit, short.class); } else if (expected == float.class && internal) { - return Cast.unboxFrom(short.class, float.class, explicit, short.class); + return PainlessCast.unboxFrom(short.class, float.class, explicit, short.class); } else if (expected == double.class && internal) { - return Cast.unboxFrom(short.class, double.class, explicit, short.class); + return PainlessCast.unboxFrom(short.class, double.class, explicit, short.class); } } else if (actual == Character.class) { if (expected == byte.class && explicit && internal) { - return Cast.unboxFrom(char.class, byte.class, true, char.class); + return PainlessCast.unboxFrom(char.class, byte.class, true, char.class); } else if (expected == short.class && explicit && internal) { - return Cast.unboxFrom(char.class, short.class, true, char.class); + return PainlessCast.unboxFrom(char.class, short.class, true, char.class); } else if (expected == char.class && internal) { - return Cast.unboxFrom(char.class, char.class, explicit, char.class); + return PainlessCast.unboxFrom(char.class, char.class, explicit, char.class); } else if (expected == int.class && internal) { - return Cast.unboxFrom(char.class, int.class, explicit, char.class); + return PainlessCast.unboxFrom(char.class, int.class, explicit, char.class); } else if (expected == long.class && internal) { - return Cast.unboxFrom(char.class, long.class, explicit, char.class); + return PainlessCast.unboxFrom(char.class, long.class, explicit, char.class); } else if (expected == float.class && internal) { - return Cast.unboxFrom(char.class, float.class, explicit, char.class); + return PainlessCast.unboxFrom(char.class, float.class, explicit, char.class); } else if (expected == double.class && internal) { - return Cast.unboxFrom(char.class, double.class, explicit, char.class); + return PainlessCast.unboxFrom(char.class, double.class, explicit, char.class); } } else if (actual == Integer.class) { if (expected == byte.class && explicit && internal) { - return Cast.unboxFrom(int.class, byte.class, true, int.class); + return PainlessCast.unboxFrom(int.class, byte.class, true, int.class); } else if (expected == short.class && explicit && internal) { - return Cast.unboxFrom(int.class, short.class, true, int.class); + return PainlessCast.unboxFrom(int.class, short.class, true, int.class); } else if (expected == char.class && explicit && internal) { - return Cast.unboxFrom(int.class, char.class, true, int.class); + return PainlessCast.unboxFrom(int.class, char.class, true, int.class); } else if (expected == int.class && internal) { - return Cast.unboxFrom(int.class, int.class, explicit, int.class); + return PainlessCast.unboxFrom(int.class, int.class, explicit, int.class); } else if (expected == long.class && internal) { - return Cast.unboxFrom(int.class, long.class, explicit, int.class); + return PainlessCast.unboxFrom(int.class, long.class, explicit, int.class); } else if (expected == float.class && internal) { - return Cast.unboxFrom(int.class, float.class, explicit, int.class); + return PainlessCast.unboxFrom(int.class, float.class, explicit, int.class); } else if (expected == double.class && internal) { - return Cast.unboxFrom(int.class, double.class, explicit, int.class); + return PainlessCast.unboxFrom(int.class, double.class, explicit, int.class); } } else if (actual == Long.class) { if (expected == byte.class && explicit && internal) { - return Cast.unboxFrom(long.class, byte.class, true, long.class); + return PainlessCast.unboxFrom(long.class, byte.class, true, long.class); } else if (expected == short.class && explicit && internal) { - return Cast.unboxFrom(long.class, short.class, true, long.class); + return PainlessCast.unboxFrom(long.class, short.class, true, long.class); } else if (expected == char.class && explicit && internal) { - return Cast.unboxFrom(long.class, char.class, true, long.class); + return PainlessCast.unboxFrom(long.class, char.class, true, long.class); } else if (expected == int.class && explicit && internal) { - return Cast.unboxFrom(long.class, int.class, true, long.class); + return PainlessCast.unboxFrom(long.class, int.class, true, long.class); } else if (expected == long.class && internal) { - return Cast.unboxFrom(long.class, long.class, explicit, long.class); + return PainlessCast.unboxFrom(long.class, long.class, explicit, long.class); } else if (expected == float.class && internal) { - return Cast.unboxFrom(long.class, float.class, explicit, long.class); + return PainlessCast.unboxFrom(long.class, float.class, explicit, long.class); } else if (expected == double.class && internal) { - return Cast.unboxFrom(long.class, double.class, explicit, long.class); + return PainlessCast.unboxFrom(long.class, double.class, explicit, long.class); } } else if (actual == Float.class) { if (expected == byte.class && explicit && internal) { - return Cast.unboxFrom(float.class, byte.class, true, float.class); + return PainlessCast.unboxFrom(float.class, byte.class, true, float.class); } else if (expected == short.class && explicit && internal) { - return Cast.unboxFrom(float.class, short.class, true, float.class); + return PainlessCast.unboxFrom(float.class, short.class, true, float.class); } else if (expected == char.class && explicit && internal) { - return Cast.unboxFrom(float.class, char.class, true, float.class); + return PainlessCast.unboxFrom(float.class, char.class, true, float.class); } else if (expected == int.class && explicit && internal) { - return Cast.unboxFrom(float.class, int.class, true, float.class); + return PainlessCast.unboxFrom(float.class, int.class, true, float.class); } else if (expected == long.class && explicit && internal) { - return Cast.unboxFrom(float.class, long.class, true, float.class); + return PainlessCast.unboxFrom(float.class, long.class, true, float.class); } else if (expected == float.class && internal) { - return Cast.unboxFrom(float.class, float.class, explicit, float.class); + return PainlessCast.unboxFrom(float.class, float.class, explicit, float.class); } else if (expected == double.class && internal) { - return Cast.unboxFrom(float.class, double.class, explicit, float.class); + return PainlessCast.unboxFrom(float.class, double.class, explicit, float.class); } } else if (actual == Double.class) { if (expected == byte.class && explicit && internal) { - return Cast.unboxFrom(double.class, byte.class, true, double.class); + return PainlessCast.unboxFrom(double.class, byte.class, true, double.class); } else if (expected == short.class && explicit && internal) { - return Cast.unboxFrom(double.class, short.class, true, double.class); + return PainlessCast.unboxFrom(double.class, short.class, true, double.class); } else if (expected == char.class && explicit && internal) { - return Cast.unboxFrom(double.class, char.class, true, double.class); + return PainlessCast.unboxFrom(double.class, char.class, true, double.class); } else if (expected == int.class && explicit && internal) { - return Cast.unboxFrom(double.class, int.class, true, double.class); + return PainlessCast.unboxFrom(double.class, int.class, true, double.class); } else if (expected == long.class && explicit && internal) { - return Cast.unboxFrom(double.class, long.class, true, double.class); + return PainlessCast.unboxFrom(double.class, long.class, true, double.class); } else if (expected == float.class && explicit && internal) { - return Cast.unboxFrom(double.class, float.class, true, double.class); + return PainlessCast.unboxFrom(double.class, float.class, true, double.class); } else if (expected == double.class && internal) { - return Cast.unboxFrom(double.class, double.class, explicit, double.class); + return PainlessCast.unboxFrom(double.class, double.class, explicit, double.class); } } @@ -462,14 +463,14 @@ public static Cast getLegalCast(Location location, Class actual, Class exp (actual != void.class && expected == def.class) || expected.isAssignableFrom(actual) || (actual.isAssignableFrom(expected) && explicit)) { - return Cast.standard(actual, expected, explicit); + return PainlessCast.standard(actual, expected, explicit); } else { throw location.createError(new ClassCastException( - "Cannot cast from [" + Definition.ClassToName(actual) + "] to [" + Definition.ClassToName(expected) + "].")); + "Cannot cast from [" + PainlessLookup.ClassToName(actual) + "] to [" + PainlessLookup.ClassToName(expected) + "].")); } } - public static Object constCast(Location location, Object constant, Cast cast) { + public static Object constCast(Location location, Object constant, PainlessCast cast) { Class fsort = cast.from; Class tsort = cast.to; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java index 8102016828c30..03345fcfff35a 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Compiler.java @@ -21,6 +21,7 @@ import org.elasticsearch.bootstrap.BootstrapInfo; import org.elasticsearch.painless.antlr.Walker; +import org.elasticsearch.painless.lookup.PainlessLookup; import org.elasticsearch.painless.node.SSource; import org.elasticsearch.painless.spi.Whitelist; import org.objectweb.asm.util.Printer; @@ -70,26 +71,26 @@ final class Compiler { */ static final class Loader extends SecureClassLoader { private final AtomicInteger lambdaCounter = new AtomicInteger(0); - private final Definition definition; + private final PainlessLookup painlessLookup; /** * @param parent The parent ClassLoader. */ - Loader(ClassLoader parent, Definition definition) { + Loader(ClassLoader parent, PainlessLookup painlessLookup) { super(parent); - this.definition = definition; + this.painlessLookup = painlessLookup; } /** * Will check to see if the {@link Class} has already been loaded when - * the {@link Definition} was initially created. Allows for {@link Whitelist}ed + * the {@link PainlessLookup} was initially created. Allows for {@link Whitelist}ed * classes to be loaded from other modules/plugins without a direct relationship * to the module's/plugin's {@link ClassLoader}. */ @Override public Class findClass(String name) throws ClassNotFoundException { - Class found = definition.getClassFromBinaryName(name); + Class found = painlessLookup.getClassFromBinaryName(name); return found != null ? found : super.findClass(name); } @@ -135,10 +136,10 @@ int newLambdaIdentifier() { /** * Return a new {@link Loader} for a script using the - * {@link Compiler}'s specified {@link Definition}. + * {@link Compiler}'s specified {@link PainlessLookup}. */ public Loader createLoader(ClassLoader parent) { - return new Loader(parent, definition); + return new Loader(parent, painlessLookup); } /** @@ -149,16 +150,16 @@ public Loader createLoader(ClassLoader parent) { /** * The whitelist the script will use. */ - private final Definition definition; + private final PainlessLookup painlessLookup; /** * Standard constructor. * @param base The class/interface the script is guaranteed to derive/implement. - * @param definition The whitelist the script will use. + * @param painlessLookup The whitelist the script will use. */ - Compiler(Class base, Definition definition) { + Compiler(Class base, PainlessLookup painlessLookup) { this.base = base; - this.definition = definition; + this.painlessLookup = painlessLookup; } /** @@ -176,10 +177,10 @@ Constructor compile(Loader loader, MainMethodReserved reserved, String name, " plugin if a script longer than this length is a requirement."); } - ScriptClassInfo scriptClassInfo = new ScriptClassInfo(definition, base); - SSource root = Walker.buildPainlessTree(scriptClassInfo, reserved, name, source, settings, definition, + ScriptClassInfo scriptClassInfo = new ScriptClassInfo(painlessLookup, base); + SSource root = Walker.buildPainlessTree(scriptClassInfo, reserved, name, source, settings, painlessLookup, null); - root.analyze(definition); + root.analyze(painlessLookup); root.write(); try { @@ -187,7 +188,7 @@ Constructor compile(Loader loader, MainMethodReserved reserved, String name, clazz.getField("$NAME").set(null, name); clazz.getField("$SOURCE").set(null, source); clazz.getField("$STATEMENTS").set(null, root.getStatements()); - clazz.getField("$DEFINITION").set(null, definition); + clazz.getField("$DEFINITION").set(null, painlessLookup); return clazz.getConstructors()[0]; } catch (Exception exception) { // Catch everything to let the user know this is something caused internally. @@ -208,10 +209,10 @@ byte[] compile(String name, String source, CompilerSettings settings, Printer de " plugin if a script longer than this length is a requirement."); } - ScriptClassInfo scriptClassInfo = new ScriptClassInfo(definition, base); - SSource root = Walker.buildPainlessTree(scriptClassInfo, new MainMethodReserved(), name, source, settings, definition, + ScriptClassInfo scriptClassInfo = new ScriptClassInfo(painlessLookup, base); + SSource root = Walker.buildPainlessTree(scriptClassInfo, new MainMethodReserved(), name, source, settings, painlessLookup, debugStream); - root.analyze(definition); + root.analyze(painlessLookup); root.write(); return root.getBytes(); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java index 8694ff7903859..f006b57f46bad 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Def.java @@ -19,13 +19,14 @@ package org.elasticsearch.painless; -import org.elasticsearch.painless.Definition.Method; -import org.elasticsearch.painless.Definition.Struct; +import org.elasticsearch.painless.lookup.PainlessLookup; +import org.elasticsearch.painless.lookup.PainlessMethod; +import org.elasticsearch.painless.lookup.PainlessClass; +import org.elasticsearch.painless.lookup.PainlessMethodKey; import java.lang.invoke.CallSite; import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; -import java.lang.invoke.MethodHandles.Lookup; import java.lang.invoke.MethodType; import java.util.BitSet; import java.util.Collections; @@ -60,14 +61,15 @@ public final class Def { */ @SuppressWarnings("unused") // getArrayLength() methods are are actually used, javac just does not know :) private static final class ArrayLengthHelper { - private static final Lookup PRIV_LOOKUP = MethodHandles.lookup(); + private static final MethodHandles.Lookup PRIVATE_METHOD_HANDLES_LOOKUP = MethodHandles.lookup(); private static final Map,MethodHandle> ARRAY_TYPE_MH_MAPPING = Collections.unmodifiableMap( Stream.of(boolean[].class, byte[].class, short[].class, int[].class, long[].class, char[].class, float[].class, double[].class, Object[].class) .collect(Collectors.toMap(Function.identity(), type -> { try { - return PRIV_LOOKUP.findStatic(PRIV_LOOKUP.lookupClass(), "getArrayLength", MethodType.methodType(int.class, type)); + return PRIVATE_METHOD_HANDLES_LOOKUP.findStatic( + PRIVATE_METHOD_HANDLES_LOOKUP.lookupClass(), "getArrayLength", MethodType.methodType(int.class, type)); } catch (ReflectiveOperationException e) { throw new AssertionError(e); } @@ -116,17 +118,17 @@ private ArrayLengthHelper() {} static final MethodHandle JAVA9_ARRAY_LENGTH_MH_FACTORY; static { - final Lookup lookup = MethodHandles.publicLookup(); + final MethodHandles.Lookup methodHandlesLookup = MethodHandles.publicLookup(); try { - MAP_GET = lookup.findVirtual(Map.class , "get", MethodType.methodType(Object.class, Object.class)); - MAP_PUT = lookup.findVirtual(Map.class , "put", MethodType.methodType(Object.class, Object.class, Object.class)); - LIST_GET = lookup.findVirtual(List.class, "get", MethodType.methodType(Object.class, int.class)); - LIST_SET = lookup.findVirtual(List.class, "set", MethodType.methodType(Object.class, int.class, Object.class)); - ITERATOR = lookup.findVirtual(Iterable.class, "iterator", MethodType.methodType(Iterator.class)); - MAP_INDEX_NORMALIZE = lookup.findStatic(Def.class, "mapIndexNormalize", + MAP_GET = methodHandlesLookup.findVirtual(Map.class , "get", MethodType.methodType(Object.class, Object.class)); + MAP_PUT = methodHandlesLookup.findVirtual(Map.class , "put", MethodType.methodType(Object.class, Object.class, Object.class)); + LIST_GET = methodHandlesLookup.findVirtual(List.class, "get", MethodType.methodType(Object.class, int.class)); + LIST_SET = methodHandlesLookup.findVirtual(List.class, "set", MethodType.methodType(Object.class, int.class, Object.class)); + ITERATOR = methodHandlesLookup.findVirtual(Iterable.class, "iterator", MethodType.methodType(Iterator.class)); + MAP_INDEX_NORMALIZE = methodHandlesLookup.findStatic(Def.class, "mapIndexNormalize", MethodType.methodType(Object.class, Map.class, Object.class)); - LIST_INDEX_NORMALIZE = lookup.findStatic(Def.class, "listIndexNormalize", + LIST_INDEX_NORMALIZE = methodHandlesLookup.findStatic(Def.class, "listIndexNormalize", MethodType.methodType(int.class, List.class, int.class)); } catch (final ReflectiveOperationException roe) { throw new AssertionError(roe); @@ -136,7 +138,7 @@ private ArrayLengthHelper() {} // https://bugs.openjdk.java.net/browse/JDK-8156915 MethodHandle arrayLengthMHFactory; try { - arrayLengthMHFactory = lookup.findStatic(MethodHandles.class, "arrayLength", + arrayLengthMHFactory = methodHandlesLookup.findStatic(MethodHandles.class, "arrayLength", MethodType.methodType(MethodHandle.class, Class.class)); } catch (final ReflectiveOperationException roe) { arrayLengthMHFactory = null; @@ -174,31 +176,31 @@ static MethodHandle arrayLengthGetter(Class arrayType) { * until it finds a matching whitelisted method. If one is not found, it throws an exception. * Otherwise it returns the matching method. *

      - * @params definition the whitelist + * @params painlessLookup the whitelist * @param receiverClass Class of the object to invoke the method on. * @param name Name of the method. * @param arity arity of method * @return matching method to invoke. never returns null. * @throws IllegalArgumentException if no matching whitelisted method was found. */ - static Method lookupMethodInternal(Definition definition, Class receiverClass, String name, int arity) { - Definition.MethodKey key = new Definition.MethodKey(name, arity); + static PainlessMethod lookupMethodInternal(PainlessLookup painlessLookup, Class receiverClass, String name, int arity) { + PainlessMethodKey key = new PainlessMethodKey(name, arity); // check whitelist for matching method for (Class clazz = receiverClass; clazz != null; clazz = clazz.getSuperclass()) { - Struct struct = definition.getPainlessStructFromJavaClass(clazz); + PainlessClass struct = painlessLookup.getPainlessStructFromJavaClass(clazz); if (struct != null) { - Method method = struct.methods.get(key); + PainlessMethod method = struct.methods.get(key); if (method != null) { return method; } } for (Class iface : clazz.getInterfaces()) { - struct = definition.getPainlessStructFromJavaClass(iface); + struct = painlessLookup.getPainlessStructFromJavaClass(iface); if (struct != null) { - Method method = struct.methods.get(key); + PainlessMethod method = struct.methods.get(key); if (method != null) { return method; } @@ -220,8 +222,8 @@ static Method lookupMethodInternal(Definition definition, Class receiverClass * until it finds a matching whitelisted method. If one is not found, it throws an exception. * Otherwise it returns a handle to the matching method. *

      - * @param definition the whitelist - * @param lookup caller's lookup + * @param painlessLookup the whitelist + * @param methodHandlesLookup caller's lookup * @param callSiteType callsite's type * @param receiverClass Class of the object to invoke the method on. * @param name Name of the method. @@ -230,13 +232,13 @@ static Method lookupMethodInternal(Definition definition, Class receiverClass * @throws IllegalArgumentException if no matching whitelisted method was found. * @throws Throwable if a method reference cannot be converted to an functional interface */ - static MethodHandle lookupMethod(Definition definition, Lookup lookup, MethodType callSiteType, - Class receiverClass, String name, Object args[]) throws Throwable { + static MethodHandle lookupMethod(PainlessLookup painlessLookup, MethodHandles.Lookup methodHandlesLookup, MethodType callSiteType, + Class receiverClass, String name, Object args[]) throws Throwable { String recipeString = (String) args[0]; int numArguments = callSiteType.parameterCount(); // simple case: no lambdas if (recipeString.isEmpty()) { - return lookupMethodInternal(definition, receiverClass, name, numArguments - 1).handle; + return lookupMethodInternal(painlessLookup, receiverClass, name, numArguments - 1).handle; } // convert recipe string to a bitset for convenience (the code below should be refactored...) @@ -259,7 +261,7 @@ static MethodHandle lookupMethod(Definition definition, Lookup lookup, MethodTyp // lookup the method with the proper arity, then we know everything (e.g. interface types of parameters). // based on these we can finally link any remaining lambdas that were deferred. - Method method = lookupMethodInternal(definition, receiverClass, name, arity); + PainlessMethod method = lookupMethodInternal(painlessLookup, receiverClass, name, arity); MethodHandle handle = method.handle; int replaced = 0; @@ -283,8 +285,8 @@ static MethodHandle lookupMethod(Definition definition, Lookup lookup, MethodTyp if (signature.charAt(0) == 'S') { // the implementation is strongly typed, now that we know the interface type, // we have everything. - filter = lookupReferenceInternal(definition, - lookup, + filter = lookupReferenceInternal(painlessLookup, + methodHandlesLookup, interfaceType, type, call, @@ -294,13 +296,13 @@ static MethodHandle lookupMethod(Definition definition, Lookup lookup, MethodTyp // this is dynamically based on the receiver type (and cached separately, underneath // this cache). It won't blow up since we never nest here (just references) MethodType nestedType = MethodType.methodType(interfaceType, captures); - CallSite nested = DefBootstrap.bootstrap(definition, - lookup, + CallSite nested = DefBootstrap.bootstrap(painlessLookup, + methodHandlesLookup, call, nestedType, 0, DefBootstrap.REFERENCE, - Definition.ClassToName(interfaceType)); + PainlessLookup.ClassToName(interfaceType)); filter = nested.dynamicInvoker(); } else { throw new AssertionError(); @@ -322,37 +324,37 @@ static MethodHandle lookupMethod(Definition definition, Lookup lookup, MethodTyp * This is just like LambdaMetaFactory, only with a dynamic type. The interface type is known, * so we simply need to lookup the matching implementation method based on receiver type. */ - static MethodHandle lookupReference(Definition definition, Lookup lookup, String interfaceClass, - Class receiverClass, String name) throws Throwable { - Class interfaceType = definition.getJavaClassFromPainlessType(interfaceClass); - Method interfaceMethod = definition.getPainlessStructFromJavaClass(interfaceType).functionalMethod; + static MethodHandle lookupReference(PainlessLookup painlessLookup, MethodHandles.Lookup methodHandlesLookup, String interfaceClass, + Class receiverClass, String name) throws Throwable { + Class interfaceType = painlessLookup.getJavaClassFromPainlessType(interfaceClass); + PainlessMethod interfaceMethod = painlessLookup.getPainlessStructFromJavaClass(interfaceType).functionalMethod; if (interfaceMethod == null) { throw new IllegalArgumentException("Class [" + interfaceClass + "] is not a functional interface"); } int arity = interfaceMethod.arguments.size(); - Method implMethod = lookupMethodInternal(definition, receiverClass, name, arity); - return lookupReferenceInternal(definition, lookup, interfaceType, implMethod.owner.name, + PainlessMethod implMethod = lookupMethodInternal(painlessLookup, receiverClass, name, arity); + return lookupReferenceInternal(painlessLookup, methodHandlesLookup, interfaceType, implMethod.owner.name, implMethod.name, receiverClass); } /** Returns a method handle to an implementation of clazz, given method reference signature. */ - private static MethodHandle lookupReferenceInternal(Definition definition, Lookup lookup, - Class clazz, String type, String call, Class... captures) + private static MethodHandle lookupReferenceInternal(PainlessLookup painlessLookup, MethodHandles.Lookup methodHandlesLookup, + Class clazz, String type, String call, Class... captures) throws Throwable { final FunctionRef ref; if ("this".equals(type)) { // user written method - Method interfaceMethod = definition.getPainlessStructFromJavaClass(clazz).functionalMethod; + PainlessMethod interfaceMethod = painlessLookup.getPainlessStructFromJavaClass(clazz).functionalMethod; if (interfaceMethod == null) { throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " + - "to [" + Definition.ClassToName(clazz) + "], not a functional interface"); + "to [" + PainlessLookup.ClassToName(clazz) + "], not a functional interface"); } int arity = interfaceMethod.arguments.size() + captures.length; final MethodHandle handle; try { - MethodHandle accessor = lookup.findStaticGetter(lookup.lookupClass(), - getUserFunctionHandleFieldName(call, arity), - MethodHandle.class); + MethodHandle accessor = methodHandlesLookup.findStaticGetter(methodHandlesLookup.lookupClass(), + getUserFunctionHandleFieldName(call, arity), + MethodHandle.class); handle = (MethodHandle)accessor.invokeExact(); } catch (NoSuchFieldException | IllegalAccessException e) { // is it a synthetic method? If we generated the method ourselves, be more helpful. It can only fail @@ -366,10 +368,10 @@ private static MethodHandle lookupReferenceInternal(Definition definition, Looku ref = new FunctionRef(clazz, interfaceMethod, call, handle.type(), captures.length); } else { // whitelist lookup - ref = new FunctionRef(definition, clazz, type, call, captures.length); + ref = new FunctionRef(painlessLookup, clazz, type, call, captures.length); } final CallSite callSite = LambdaBootstrap.lambdaBootstrap( - lookup, + methodHandlesLookup, ref.interfaceMethodName, ref.factoryMethodType, ref.interfaceMethodType, @@ -407,16 +409,16 @@ public static String getUserFunctionHandleFieldName(String name, int arity) { * until it finds a matching whitelisted getter. If one is not found, it throws an exception. * Otherwise it returns a handle to the matching getter. *

      - * @param definition the whitelist + * @param painlessLookup the whitelist * @param receiverClass Class of the object to retrieve the field from. * @param name Name of the field. * @return pointer to matching field. never returns null. * @throws IllegalArgumentException if no matching whitelisted field was found. */ - static MethodHandle lookupGetter(Definition definition, Class receiverClass, String name) { + static MethodHandle lookupGetter(PainlessLookup painlessLookup, Class receiverClass, String name) { // first try whitelist for (Class clazz = receiverClass; clazz != null; clazz = clazz.getSuperclass()) { - Struct struct = definition.getPainlessStructFromJavaClass(clazz); + PainlessClass struct = painlessLookup.getPainlessStructFromJavaClass(clazz); if (struct != null) { MethodHandle handle = struct.getters.get(name); @@ -426,7 +428,7 @@ static MethodHandle lookupGetter(Definition definition, Class receiverClass, } for (final Class iface : clazz.getInterfaces()) { - struct = definition.getPainlessStructFromJavaClass(iface); + struct = painlessLookup.getPainlessStructFromJavaClass(iface); if (struct != null) { MethodHandle handle = struct.getters.get(name); @@ -478,16 +480,16 @@ static MethodHandle lookupGetter(Definition definition, Class receiverClass, * until it finds a matching whitelisted setter. If one is not found, it throws an exception. * Otherwise it returns a handle to the matching setter. *

      - * @param definition the whitelist + * @param painlessLookup the whitelist * @param receiverClass Class of the object to retrieve the field from. * @param name Name of the field. * @return pointer to matching field. never returns null. * @throws IllegalArgumentException if no matching whitelisted field was found. */ - static MethodHandle lookupSetter(Definition definition, Class receiverClass, String name) { + static MethodHandle lookupSetter(PainlessLookup painlessLookup, Class receiverClass, String name) { // first try whitelist for (Class clazz = receiverClass; clazz != null; clazz = clazz.getSuperclass()) { - Struct struct = definition.getPainlessStructFromJavaClass(clazz); + PainlessClass struct = painlessLookup.getPainlessStructFromJavaClass(clazz); if (struct != null) { MethodHandle handle = struct.setters.get(name); @@ -497,7 +499,7 @@ static MethodHandle lookupSetter(Definition definition, Class receiverClass, } for (final Class iface : clazz.getInterfaces()) { - struct = definition.getPainlessStructFromJavaClass(iface); + struct = painlessLookup.getPainlessStructFromJavaClass(iface); if (struct != null) { MethodHandle handle = struct.setters.get(name); @@ -592,14 +594,15 @@ static MethodHandle lookupArrayLoad(Class receiverClass) { */ @SuppressWarnings("unused") // iterator() methods are are actually used, javac just does not know :) private static final class ArrayIteratorHelper { - private static final Lookup PRIV_LOOKUP = MethodHandles.lookup(); + private static final MethodHandles.Lookup PRIVATE_METHOD_HANDLES_LOOKUP = MethodHandles.lookup(); private static final Map,MethodHandle> ARRAY_TYPE_MH_MAPPING = Collections.unmodifiableMap( Stream.of(boolean[].class, byte[].class, short[].class, int[].class, long[].class, char[].class, float[].class, double[].class, Object[].class) .collect(Collectors.toMap(Function.identity(), type -> { try { - return PRIV_LOOKUP.findStatic(PRIV_LOOKUP.lookupClass(), "iterator", MethodType.methodType(Iterator.class, type)); + return PRIVATE_METHOD_HANDLES_LOOKUP.findStatic( + PRIVATE_METHOD_HANDLES_LOOKUP.lookupClass(), "iterator", MethodType.methodType(Iterator.class, type)); } catch (ReflectiveOperationException e) { throw new AssertionError(e); } @@ -860,14 +863,14 @@ public static int listIndexNormalize(final List value, int index) { */ @SuppressWarnings("unused") // normalizeIndex() methods are are actually used, javac just does not know :) private static final class ArrayIndexNormalizeHelper { - private static final Lookup PRIV_LOOKUP = MethodHandles.lookup(); + private static final MethodHandles.Lookup PRIVATE_METHOD_HANDLES_LOOKUP = MethodHandles.lookup(); private static final Map,MethodHandle> ARRAY_TYPE_MH_MAPPING = Collections.unmodifiableMap( Stream.of(boolean[].class, byte[].class, short[].class, int[].class, long[].class, char[].class, float[].class, double[].class, Object[].class) .collect(Collectors.toMap(Function.identity(), type -> { try { - return PRIV_LOOKUP.findStatic(PRIV_LOOKUP.lookupClass(), "normalizeIndex", + return PRIVATE_METHOD_HANDLES_LOOKUP.findStatic(PRIVATE_METHOD_HANDLES_LOOKUP.lookupClass(), "normalizeIndex", MethodType.methodType(int.class, type, int.class)); } catch (ReflectiveOperationException e) { throw new AssertionError(e); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java index 9c7c7f631b68d..2fadaf30964a6 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefBootstrap.java @@ -20,11 +20,11 @@ package org.elasticsearch.painless; import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.painless.lookup.PainlessLookup; import java.lang.invoke.CallSite; import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; -import java.lang.invoke.MethodHandles.Lookup; import java.lang.invoke.MethodType; import java.lang.invoke.MutableCallSite; import java.lang.invoke.WrongMethodTypeException; @@ -104,20 +104,21 @@ static final class PIC extends MutableCallSite { /** maximum number of types before we go megamorphic */ static final int MAX_DEPTH = 5; - private final Definition definition; - private final Lookup lookup; + private final PainlessLookup painlessLookup; + private final MethodHandles.Lookup methodHandlesLookup; private final String name; private final int flavor; private final Object[] args; int depth; // pkg-protected for testing - PIC(Definition definition, Lookup lookup, String name, MethodType type, int initialDepth, int flavor, Object[] args) { + PIC(PainlessLookup painlessLookup, MethodHandles.Lookup methodHandlesLookup, + String name, MethodType type, int initialDepth, int flavor, Object[] args) { super(type); if (type.parameterType(0) != Object.class) { throw new BootstrapMethodError("The receiver type (1st arg) of invokedynamic descriptor must be Object."); } - this.definition = definition; - this.lookup = lookup; + this.painlessLookup = painlessLookup; + this.methodHandlesLookup = methodHandlesLookup; this.name = name; this.flavor = flavor; this.args = args; @@ -144,11 +145,11 @@ static boolean checkClass(Class clazz, Object receiver) { private MethodHandle lookup(int flavor, String name, Class receiver) throws Throwable { switch(flavor) { case METHOD_CALL: - return Def.lookupMethod(definition, lookup, type(), receiver, name, args); + return Def.lookupMethod(painlessLookup, methodHandlesLookup, type(), receiver, name, args); case LOAD: - return Def.lookupGetter(definition, receiver, name); + return Def.lookupGetter(painlessLookup, receiver, name); case STORE: - return Def.lookupSetter(definition, receiver, name); + return Def.lookupSetter(painlessLookup, receiver, name); case ARRAY_LOAD: return Def.lookupArrayLoad(receiver); case ARRAY_STORE: @@ -156,7 +157,7 @@ private MethodHandle lookup(int flavor, String name, Class receiver) throws T case ITERATOR: return Def.lookupIterator(receiver); case REFERENCE: - return Def.lookupReference(definition, lookup, (String) args[0], receiver, name); + return Def.lookupReference(painlessLookup, methodHandlesLookup, (String) args[0], receiver, name); case INDEX_NORMALIZE: return Def.lookupIndexNormalize(receiver); default: throw new AssertionError(); @@ -216,17 +217,17 @@ Object fallback(final Object[] callArgs) throws Throwable { private static final MethodHandle FALLBACK; private static final MethodHandle MEGAMORPHIC_LOOKUP; static { - final Lookup lookup = MethodHandles.lookup(); - final Lookup publicLookup = MethodHandles.publicLookup(); + final MethodHandles.Lookup methodHandlesLookup = MethodHandles.lookup(); + final MethodHandles.Lookup publicMethodHandlesLookup = MethodHandles.publicLookup(); try { - CHECK_CLASS = lookup.findStatic(lookup.lookupClass(), "checkClass", - MethodType.methodType(boolean.class, Class.class, Object.class)); - FALLBACK = lookup.findVirtual(lookup.lookupClass(), "fallback", + CHECK_CLASS = methodHandlesLookup.findStatic(methodHandlesLookup.lookupClass(), "checkClass", + MethodType.methodType(boolean.class, Class.class, Object.class)); + FALLBACK = methodHandlesLookup.findVirtual(methodHandlesLookup.lookupClass(), "fallback", MethodType.methodType(Object.class, Object[].class)); - MethodHandle mh = publicLookup.findVirtual(ClassValue.class, "get", + MethodHandle mh = publicMethodHandlesLookup.findVirtual(ClassValue.class, "get", MethodType.methodType(Object.class, Class.class)); mh = MethodHandles.filterArguments(mh, 1, - publicLookup.findVirtual(Object.class, "getClass", MethodType.methodType(Class.class))); + publicMethodHandlesLookup.findVirtual(Object.class, "getClass", MethodType.methodType(Class.class))); MEGAMORPHIC_LOOKUP = mh.asType(mh.type().changeReturnType(MethodHandle.class)); } catch (ReflectiveOperationException e) { throw new AssertionError(e); @@ -402,16 +403,16 @@ static boolean checkBoth(Class left, Class right, Object leftObject, Objec private static final MethodHandle CHECK_BOTH; private static final MethodHandle FALLBACK; static { - final Lookup lookup = MethodHandles.lookup(); + final MethodHandles.Lookup methodHandlesLookup = MethodHandles.lookup(); try { - CHECK_LHS = lookup.findStatic(lookup.lookupClass(), "checkLHS", - MethodType.methodType(boolean.class, Class.class, Object.class)); - CHECK_RHS = lookup.findStatic(lookup.lookupClass(), "checkRHS", - MethodType.methodType(boolean.class, Class.class, Class.class, Object.class, Object.class)); - CHECK_BOTH = lookup.findStatic(lookup.lookupClass(), "checkBoth", - MethodType.methodType(boolean.class, Class.class, Class.class, Object.class, Object.class)); - FALLBACK = lookup.findVirtual(lookup.lookupClass(), "fallback", - MethodType.methodType(Object.class, Object[].class)); + CHECK_LHS = methodHandlesLookup.findStatic(methodHandlesLookup.lookupClass(), "checkLHS", + MethodType.methodType(boolean.class, Class.class, Object.class)); + CHECK_RHS = methodHandlesLookup.findStatic(methodHandlesLookup.lookupClass(), "checkRHS", + MethodType.methodType(boolean.class, Class.class, Class.class, Object.class, Object.class)); + CHECK_BOTH = methodHandlesLookup.findStatic(methodHandlesLookup.lookupClass(), "checkBoth", + MethodType.methodType(boolean.class, Class.class, Class.class, Object.class, Object.class)); + FALLBACK = methodHandlesLookup.findVirtual(methodHandlesLookup.lookupClass(), "fallback", + MethodType.methodType(Object.class, Object[].class)); } catch (ReflectiveOperationException e) { throw new AssertionError(e); } @@ -427,12 +428,12 @@ static boolean checkBoth(Class left, Class right, Object leftObject, Objec *

    • {@code flavor}: type of dynamic call it is (and which part of whitelist to look at). *
    • {@code args}: flavor-specific args. *
    - * And we take the {@link Definition} used to compile the script for whitelist checking. + * And we take the {@link PainlessLookup} used to compile the script for whitelist checking. *

    * see https://docs.oracle.com/javase/specs/jvms/se7/html/jvms-6.html#jvms-6.5.invokedynamic */ - public static CallSite bootstrap(Definition definition, Lookup lookup, String name, MethodType type, int initialDepth, int flavor, - Object... args) { + public static CallSite bootstrap(PainlessLookup painlessLookup, MethodHandles.Lookup methodHandlesLookup, String name, + MethodType type, int initialDepth, int flavor, Object... args) { // validate arguments switch(flavor) { // "function-call" like things get a polymorphic cache @@ -451,7 +452,7 @@ public static CallSite bootstrap(Definition definition, Lookup lookup, String na if (args.length != numLambdas + 1) { throw new BootstrapMethodError("Illegal number of parameters: expected " + numLambdas + " references"); } - return new PIC(definition, lookup, name, type, initialDepth, flavor, args); + return new PIC(painlessLookup, methodHandlesLookup, name, type, initialDepth, flavor, args); case LOAD: case STORE: case ARRAY_LOAD: @@ -461,7 +462,7 @@ public static CallSite bootstrap(Definition definition, Lookup lookup, String na if (args.length > 0) { throw new BootstrapMethodError("Illegal static bootstrap parameters for flavor: " + flavor); } - return new PIC(definition, lookup, name, type, initialDepth, flavor, args); + return new PIC(painlessLookup, methodHandlesLookup, name, type, initialDepth, flavor, args); case REFERENCE: if (args.length != 1) { throw new BootstrapMethodError("Invalid number of parameters for reference call"); @@ -469,7 +470,7 @@ public static CallSite bootstrap(Definition definition, Lookup lookup, String na if (args[0] instanceof String == false) { throw new BootstrapMethodError("Illegal parameter for reference call: " + args[0]); } - return new PIC(definition, lookup, name, type, initialDepth, flavor, args); + return new PIC(painlessLookup, methodHandlesLookup, name, type, initialDepth, flavor, args); // operators get monomorphic cache, with a generic impl for a fallback case UNARY_OPERATOR: diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefMath.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefMath.java index f903c0571b2bd..01d627e3962a2 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefMath.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/DefMath.java @@ -21,7 +21,6 @@ import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; -import java.lang.invoke.MethodHandles.Lookup; import java.lang.invoke.MethodType; import java.util.Collections; import java.util.HashMap; @@ -1070,7 +1069,7 @@ private static Class promote(Class a, Class b) { } } - private static final Lookup PRIV_LOOKUP = MethodHandles.lookup(); + private static final MethodHandles.Lookup PRIVATE_METHOD_HANDLES_LOOKUP = MethodHandles.lookup(); private static final Map,Map> TYPE_OP_MAPPING = Collections.unmodifiableMap( Stream.of(boolean.class, int.class, long.class, float.class, double.class, Object.class) @@ -1081,26 +1080,26 @@ private static Class promote(Class a, Class b) { MethodType binary = MethodType.methodType(type, type, type); MethodType comparison = MethodType.methodType(boolean.class, type, type); MethodType shift = MethodType.methodType(type, type, long.class); - Class clazz = PRIV_LOOKUP.lookupClass(); - map.put("not", PRIV_LOOKUP.findStatic(clazz, "not", unary)); - map.put("neg", PRIV_LOOKUP.findStatic(clazz, "neg", unary)); - map.put("plus", PRIV_LOOKUP.findStatic(clazz, "plus", unary)); - map.put("mul", PRIV_LOOKUP.findStatic(clazz, "mul", binary)); - map.put("div", PRIV_LOOKUP.findStatic(clazz, "div", binary)); - map.put("rem", PRIV_LOOKUP.findStatic(clazz, "rem", binary)); - map.put("add", PRIV_LOOKUP.findStatic(clazz, "add", binary)); - map.put("sub", PRIV_LOOKUP.findStatic(clazz, "sub", binary)); - map.put("and", PRIV_LOOKUP.findStatic(clazz, "and", binary)); - map.put("or", PRIV_LOOKUP.findStatic(clazz, "or", binary)); - map.put("xor", PRIV_LOOKUP.findStatic(clazz, "xor", binary)); - map.put("eq", PRIV_LOOKUP.findStatic(clazz, "eq", comparison)); - map.put("lt", PRIV_LOOKUP.findStatic(clazz, "lt", comparison)); - map.put("lte", PRIV_LOOKUP.findStatic(clazz, "lte", comparison)); - map.put("gt", PRIV_LOOKUP.findStatic(clazz, "gt", comparison)); - map.put("gte", PRIV_LOOKUP.findStatic(clazz, "gte", comparison)); - map.put("lsh", PRIV_LOOKUP.findStatic(clazz, "lsh", shift)); - map.put("rsh", PRIV_LOOKUP.findStatic(clazz, "rsh", shift)); - map.put("ush", PRIV_LOOKUP.findStatic(clazz, "ush", shift)); + Class clazz = PRIVATE_METHOD_HANDLES_LOOKUP.lookupClass(); + map.put("not", PRIVATE_METHOD_HANDLES_LOOKUP.findStatic(clazz, "not", unary)); + map.put("neg", PRIVATE_METHOD_HANDLES_LOOKUP.findStatic(clazz, "neg", unary)); + map.put("plus", PRIVATE_METHOD_HANDLES_LOOKUP.findStatic(clazz, "plus", unary)); + map.put("mul", PRIVATE_METHOD_HANDLES_LOOKUP.findStatic(clazz, "mul", binary)); + map.put("div", PRIVATE_METHOD_HANDLES_LOOKUP.findStatic(clazz, "div", binary)); + map.put("rem", PRIVATE_METHOD_HANDLES_LOOKUP.findStatic(clazz, "rem", binary)); + map.put("add", PRIVATE_METHOD_HANDLES_LOOKUP.findStatic(clazz, "add", binary)); + map.put("sub", PRIVATE_METHOD_HANDLES_LOOKUP.findStatic(clazz, "sub", binary)); + map.put("and", PRIVATE_METHOD_HANDLES_LOOKUP.findStatic(clazz, "and", binary)); + map.put("or", PRIVATE_METHOD_HANDLES_LOOKUP.findStatic(clazz, "or", binary)); + map.put("xor", PRIVATE_METHOD_HANDLES_LOOKUP.findStatic(clazz, "xor", binary)); + map.put("eq", PRIVATE_METHOD_HANDLES_LOOKUP.findStatic(clazz, "eq", comparison)); + map.put("lt", PRIVATE_METHOD_HANDLES_LOOKUP.findStatic(clazz, "lt", comparison)); + map.put("lte", PRIVATE_METHOD_HANDLES_LOOKUP.findStatic(clazz, "lte", comparison)); + map.put("gt", PRIVATE_METHOD_HANDLES_LOOKUP.findStatic(clazz, "gt", comparison)); + map.put("gte", PRIVATE_METHOD_HANDLES_LOOKUP.findStatic(clazz, "gte", comparison)); + map.put("lsh", PRIVATE_METHOD_HANDLES_LOOKUP.findStatic(clazz, "lsh", shift)); + map.put("rsh", PRIVATE_METHOD_HANDLES_LOOKUP.findStatic(clazz, "rsh", shift)); + map.put("ush", PRIVATE_METHOD_HANDLES_LOOKUP.findStatic(clazz, "ush", shift)); return map; } catch (ReflectiveOperationException e) { throw new AssertionError(e); @@ -1188,14 +1187,14 @@ static Number getNumber(Object o) { private static final MethodHandle DYNAMIC_CAST; private static final MethodHandle DYNAMIC_RECEIVER_CAST; static { - final Lookup lookup = MethodHandles.lookup(); + final MethodHandles.Lookup methodHandlesLookup = MethodHandles.lookup(); try { - DYNAMIC_CAST = lookup.findStatic(lookup.lookupClass(), - "dynamicCast", - MethodType.methodType(Object.class, Class.class, Object.class)); - DYNAMIC_RECEIVER_CAST = lookup.findStatic(lookup.lookupClass(), - "dynamicReceiverCast", - MethodType.methodType(Object.class, Object.class, Object.class)); + DYNAMIC_CAST = methodHandlesLookup.findStatic(methodHandlesLookup.lookupClass(), + "dynamicCast", + MethodType.methodType(Object.class, Class.class, Object.class)); + DYNAMIC_RECEIVER_CAST = methodHandlesLookup.findStatic(methodHandlesLookup.lookupClass(), + "dynamicReceiverCast", + MethodType.methodType(Object.class, Object.class, Object.class)); } catch (ReflectiveOperationException e) { throw new AssertionError(e); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java index 1b438965538ce..6ab7292c7f65b 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/FunctionRef.java @@ -19,7 +19,10 @@ package org.elasticsearch.painless; -import org.elasticsearch.painless.Definition.Method; +import org.elasticsearch.painless.lookup.PainlessClass; +import org.elasticsearch.painless.lookup.PainlessLookup; +import org.elasticsearch.painless.lookup.PainlessMethod; +import org.elasticsearch.painless.lookup.PainlessMethodKey; import org.objectweb.asm.Type; import java.lang.invoke.MethodType; @@ -55,9 +58,9 @@ public class FunctionRef { public final MethodType delegateMethodType; /** interface method */ - public final Method interfaceMethod; + public final PainlessMethod interfaceMethod; /** delegate method */ - public final Method delegateMethod; + public final PainlessMethod delegateMethod; /** factory method type descriptor */ public final String factoryDescriptor; @@ -71,15 +74,15 @@ public class FunctionRef { /** * Creates a new FunctionRef, which will resolve {@code type::call} from the whitelist. - * @param definition the whitelist against which this script is being compiled + * @param painlessLookup the whitelist against which this script is being compiled * @param expected functional interface type to implement. * @param type the left hand side of a method reference expression * @param call the right hand side of a method reference expression * @param numCaptures number of captured arguments */ - public FunctionRef(Definition definition, Class expected, String type, String call, int numCaptures) { - this(expected, definition.getPainlessStructFromJavaClass(expected).functionalMethod, - lookup(definition, expected, type, call, numCaptures > 0), numCaptures); + public FunctionRef(PainlessLookup painlessLookup, Class expected, String type, String call, int numCaptures) { + this(expected, painlessLookup.getPainlessStructFromJavaClass(expected).functionalMethod, + lookup(painlessLookup, expected, type, call, numCaptures > 0), numCaptures); } /** @@ -89,7 +92,7 @@ public FunctionRef(Definition definition, Class expected, String type, String * @param delegateMethod implementation method * @param numCaptures number of captured arguments */ - public FunctionRef(Class expected, Method interfaceMethod, Method delegateMethod, int numCaptures) { + public FunctionRef(Class expected, PainlessMethod interfaceMethod, PainlessMethod delegateMethod, int numCaptures) { MethodType delegateMethodType = delegateMethod.getMethodType(); interfaceMethodName = interfaceMethod.name; @@ -135,7 +138,7 @@ public FunctionRef(Class expected, Method interfaceMethod, Method delegateMet * It is for runtime use only. */ public FunctionRef(Class expected, - Method interfaceMethod, String delegateMethodName, MethodType delegateMethodType, int numCaptures) { + PainlessMethod interfaceMethod, String delegateMethodName, MethodType delegateMethodType, int numCaptures) { interfaceMethodName = interfaceMethod.name; factoryMethodType = MethodType.methodType(expected, delegateMethodType.dropParameterTypes(numCaptures, delegateMethodType.parameterCount())); @@ -158,25 +161,25 @@ public FunctionRef(Class expected, /** * Looks up {@code type::call} from the whitelist, and returns a matching method. */ - private static Definition.Method lookup(Definition definition, Class expected, - String type, String call, boolean receiverCaptured) { + private static PainlessMethod lookup(PainlessLookup painlessLookup, Class expected, + String type, String call, boolean receiverCaptured) { // check its really a functional interface // for e.g. Comparable - Method method = definition.getPainlessStructFromJavaClass(expected).functionalMethod; + PainlessMethod method = painlessLookup.getPainlessStructFromJavaClass(expected).functionalMethod; if (method == null) { throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " + - "to [" + Definition.ClassToName(expected) + "], not a functional interface"); + "to [" + PainlessLookup.ClassToName(expected) + "], not a functional interface"); } // lookup requested method - Definition.Struct struct = definition.getPainlessStructFromJavaClass(definition.getJavaClassFromPainlessType(type)); - final Definition.Method impl; + PainlessClass struct = painlessLookup.getPainlessStructFromJavaClass(painlessLookup.getJavaClassFromPainlessType(type)); + final PainlessMethod impl; // ctor ref if ("new".equals(call)) { - impl = struct.constructors.get(new Definition.MethodKey("", method.arguments.size())); + impl = struct.constructors.get(new PainlessMethodKey("", method.arguments.size())); } else { // look for a static impl first - Definition.Method staticImpl = struct.staticMethods.get(new Definition.MethodKey(call, method.arguments.size())); + PainlessMethod staticImpl = struct.staticMethods.get(new PainlessMethodKey(call, method.arguments.size())); if (staticImpl == null) { // otherwise a virtual impl final int arity; @@ -187,7 +190,7 @@ private static Definition.Method lookup(Definition definition, Class expected // receiver passed arity = method.arguments.size() - 1; } - impl = struct.methods.get(new Definition.MethodKey(call, arity)); + impl = struct.methods.get(new PainlessMethodKey(call, arity)); } else { impl = staticImpl; } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java index 2498396c891d1..bab1b7942d657 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/Locals.java @@ -19,8 +19,9 @@ package org.elasticsearch.painless; -import org.elasticsearch.painless.Definition.Method; -import org.elasticsearch.painless.Definition.MethodKey; +import org.elasticsearch.painless.lookup.PainlessLookup; +import org.elasticsearch.painless.lookup.PainlessMethod; +import org.elasticsearch.painless.lookup.PainlessMethodKey; import org.elasticsearch.painless.ScriptClassInfo.MethodArgument; import java.util.Arrays; @@ -59,7 +60,7 @@ public static Locals newLocalScope(Locals currentScope) { */ public static Locals newLambdaScope(Locals programScope, Class returnType, List parameters, int captureCount, int maxLoopCounter) { - Locals locals = new Locals(programScope, programScope.definition, returnType, KEYWORDS); + Locals locals = new Locals(programScope, programScope.painlessLookup, returnType, KEYWORDS); for (int i = 0; i < parameters.size(); i++) { Parameter parameter = parameters.get(i); // TODO: allow non-captures to be r/w: @@ -78,7 +79,7 @@ public static Locals newLambdaScope(Locals programScope, Class returnType, Li /** Creates a new function scope inside the current scope */ public static Locals newFunctionScope(Locals programScope, Class returnType, List parameters, int maxLoopCounter) { - Locals locals = new Locals(programScope, programScope.definition, returnType, KEYWORDS); + Locals locals = new Locals(programScope, programScope.painlessLookup, returnType, KEYWORDS); for (Parameter parameter : parameters) { locals.addVariable(parameter.location, parameter.clazz, parameter.name, false); } @@ -92,7 +93,7 @@ public static Locals newFunctionScope(Locals programScope, Class returnType, /** Creates a new main method scope */ public static Locals newMainMethodScope(ScriptClassInfo scriptClassInfo, Locals programScope, int maxLoopCounter) { Locals locals = new Locals( - programScope, programScope.definition, scriptClassInfo.getExecuteMethodReturnType(), KEYWORDS); + programScope, programScope.painlessLookup, scriptClassInfo.getExecuteMethodReturnType(), KEYWORDS); // This reference. Internal use only. locals.defineVariable(null, Object.class, THIS, true); @@ -109,9 +110,9 @@ public static Locals newMainMethodScope(ScriptClassInfo scriptClassInfo, Locals } /** Creates a new program scope: the list of methods. It is the parent for all methods */ - public static Locals newProgramScope(Definition definition, Collection methods) { - Locals locals = new Locals(null, definition, null, null); - for (Method method : methods) { + public static Locals newProgramScope(PainlessLookup painlessLookup, Collection methods) { + Locals locals = new Locals(null, painlessLookup, null, null); + for (PainlessMethod method : methods) { locals.addMethod(method); } return locals; @@ -142,8 +143,8 @@ public Variable getVariable(Location location, String name) { } /** Looks up a method. Returns null if the method does not exist. */ - public Method getMethod(MethodKey key) { - Method method = lookupMethod(key); + public PainlessMethod getMethod(PainlessMethodKey key) { + PainlessMethod method = lookupMethod(key); if (method != null) { return method; } @@ -179,14 +180,14 @@ public Locals getProgramScope() { } /** Whitelist against which this script is being compiled. */ - public Definition getDefinition() { - return definition; + public PainlessLookup getPainlessLookup() { + return painlessLookup; } ///// private impl /** Whitelist against which this script is being compiled. */ - private final Definition definition; + private final PainlessLookup painlessLookup; // parent scope private final Locals parent; // return type of this scope @@ -198,21 +199,21 @@ public Definition getDefinition() { // variable name -> variable private Map variables; // method name+arity -> methods - private Map methods; + private Map methods; /** * Create a new Locals */ private Locals(Locals parent) { - this(parent, parent.definition, parent.returnType, parent.keywords); + this(parent, parent.painlessLookup, parent.returnType, parent.keywords); } /** * Create a new Locals with specified return type */ - private Locals(Locals parent, Definition definition, Class returnType, Set keywords) { + private Locals(Locals parent, PainlessLookup painlessLookup, Class returnType, Set keywords) { this.parent = parent; - this.definition = definition; + this.painlessLookup = painlessLookup; this.returnType = returnType; this.keywords = keywords; if (parent == null) { @@ -236,7 +237,7 @@ private Variable lookupVariable(Location location, String name) { } /** Looks up a method at this scope only. Returns null if the method does not exist. */ - private Method lookupMethod(MethodKey key) { + private PainlessMethod lookupMethod(PainlessMethodKey key) { if (methods == null) { return null; } @@ -255,11 +256,11 @@ private Variable defineVariable(Location location, Class type, String name, b return variable; } - private void addMethod(Method method) { + private void addMethod(PainlessMethod method) { if (methods == null) { methods = new HashMap<>(); } - methods.put(new MethodKey(method.name, method.arguments.size()), method); + methods.put(new PainlessMethodKey(method.name, method.arguments.size()), method); // TODO: check result } @@ -291,7 +292,7 @@ public int getSlot() { @Override public String toString() { StringBuilder b = new StringBuilder(); - b.append("Variable[type=").append(Definition.ClassToName(clazz)); + b.append("Variable[type=").append(PainlessLookup.ClassToName(clazz)); b.append(",name=").append(name); b.append(",slot=").append(slot); if (readonly) { diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java index 5167f7d1434de..e0a780d418843 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/MethodWriter.java @@ -19,8 +19,8 @@ package org.elasticsearch.painless; -import org.elasticsearch.painless.Definition.Cast; -import org.elasticsearch.painless.Definition.def; +import org.elasticsearch.painless.lookup.PainlessCast; +import org.elasticsearch.painless.lookup.PainlessLookup.def; import org.objectweb.asm.ClassVisitor; import org.objectweb.asm.Label; import org.objectweb.asm.Opcodes; @@ -130,7 +130,7 @@ public void writeLoopCounter(int slot, int count, Location location) { mark(end); } - public void writeCast(Cast cast) { + public void writeCast(PainlessCast cast) { if (cast != null) { if (cast.from == char.class && cast.to == String.class) { invokeStatic(UTILITY_TYPE, CHAR_TO_STRING); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessExplainError.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessExplainError.java index 7ae93eba22632..1236c4977e802 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessExplainError.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessExplainError.java @@ -20,6 +20,8 @@ package org.elasticsearch.painless; import org.elasticsearch.painless.api.Debug; +import org.elasticsearch.painless.lookup.PainlessClass; +import org.elasticsearch.painless.lookup.PainlessLookup; import org.elasticsearch.script.ScriptException; import java.util.List; @@ -46,7 +48,7 @@ Object getObjectToExplain() { /** * Headers to be added to the {@link ScriptException} for structured rendering. */ - public Map> getHeaders(Definition definition) { + public Map> getHeaders(PainlessLookup painlessLookup) { Map> headers = new TreeMap<>(); String toString = "null"; String javaClassName = null; @@ -54,7 +56,7 @@ public Map> getHeaders(Definition definition) { if (objectToExplain != null) { toString = objectToExplain.toString(); javaClassName = objectToExplain.getClass().getName(); - Definition.Struct struct = definition.getPainlessStructFromJavaClass(objectToExplain.getClass()); + PainlessClass struct = painlessLookup.getPainlessStructFromJavaClass(objectToExplain.getClass()); if (struct != null) { painlessClassName = struct.name; } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java index 339e58c763c78..ae1944c9bd3a9 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/PainlessScriptEngine.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.component.AbstractComponent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.painless.Compiler.Loader; +import org.elasticsearch.painless.lookup.PainlessLookup; import org.elasticsearch.painless.spi.Whitelist; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.ScriptContext; @@ -101,9 +102,9 @@ public PainlessScriptEngine(Settings settings, Map, List, List> entry : contexts.entrySet()) { ScriptContext context = entry.getKey(); if (context.instanceClazz.equals(SearchScript.class) || context.instanceClazz.equals(ExecutableScript.class)) { - contextsToCompilers.put(context, new Compiler(GenericElasticsearchScript.class, new Definition(entry.getValue()))); + contextsToCompilers.put(context, new Compiler(GenericElasticsearchScript.class, new PainlessLookup(entry.getValue()))); } else { - contextsToCompilers.put(context, new Compiler(context.instanceClazz, new Definition(entry.getValue()))); + contextsToCompilers.put(context, new Compiler(context.instanceClazz, new PainlessLookup(entry.getValue()))); } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptClassInfo.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptClassInfo.java index 0ec806282db2f..df43dba7b3476 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptClassInfo.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/ScriptClassInfo.java @@ -19,6 +19,8 @@ package org.elasticsearch.painless; +import org.elasticsearch.painless.lookup.PainlessLookup; + import java.lang.invoke.MethodType; import java.lang.reflect.Field; import java.lang.reflect.Modifier; @@ -42,7 +44,7 @@ public class ScriptClassInfo { private final List getMethods; private final List> getReturns; - public ScriptClassInfo(Definition definition, Class baseClass) { + public ScriptClassInfo(PainlessLookup painlessLookup, Class baseClass) { this.baseClass = baseClass; // Find the main method and the uses$argName methods @@ -68,8 +70,9 @@ public ScriptClassInfo(Definition definition, Class baseClass) { } if (m.getName().startsWith("get") && m.getName().equals("getClass") == false && Modifier.isStatic(m.getModifiers()) == false) { getReturns.add( - definitionTypeForClass(definition, m.getReturnType(), componentType -> "[" + m.getName() + "] has unknown return type [" - + componentType.getName() + "]. Painless can only support getters with return types that are whitelisted.")); + definitionTypeForClass(painlessLookup, m.getReturnType(), componentType -> "[" + m.getName() + "] has unknown return " + + "type [" + componentType.getName() + "]. Painless can only support getters with return types that are " + + "whitelisted.")); getMethods.add(new org.objectweb.asm.commons.Method(m.getName(), MethodType.methodType(m.getReturnType()).toMethodDescriptorString())); @@ -78,7 +81,7 @@ public ScriptClassInfo(Definition definition, Class baseClass) { } MethodType methodType = MethodType.methodType(executeMethod.getReturnType(), executeMethod.getParameterTypes()); this.executeMethod = new org.objectweb.asm.commons.Method(executeMethod.getName(), methodType.toMethodDescriptorString()); - executeMethodReturnType = definitionTypeForClass(definition, executeMethod.getReturnType(), + executeMethodReturnType = definitionTypeForClass(painlessLookup, executeMethod.getReturnType(), componentType -> "Painless can only implement execute methods returning a whitelisted type but [" + baseClass.getName() + "#execute] returns [" + componentType.getName() + "] which isn't whitelisted."); @@ -91,7 +94,7 @@ public ScriptClassInfo(Definition definition, Class baseClass) { + baseClass.getName() + "#execute] takes [1] argument."); } for (int arg = 0; arg < types.length; arg++) { - arguments.add(methodArgument(definition, types[arg], argumentNamesConstant[arg])); + arguments.add(methodArgument(painlessLookup, types[arg], argumentNamesConstant[arg])); } this.executeArguments = unmodifiableList(arguments); this.needsMethods = unmodifiableList(needsMethods); @@ -171,22 +174,22 @@ public String getName() { } } - private MethodArgument methodArgument(Definition definition, Class clazz, String argName) { - Class defClass = definitionTypeForClass(definition, clazz, componentType -> "[" + argName + "] is of unknown type [" + private MethodArgument methodArgument(PainlessLookup painlessLookup, Class clazz, String argName) { + Class defClass = definitionTypeForClass(painlessLookup, clazz, componentType -> "[" + argName + "] is of unknown type [" + componentType.getName() + ". Painless interfaces can only accept arguments that are of whitelisted types."); return new MethodArgument(defClass, argName); } - private static Class definitionTypeForClass(Definition definition, Class type, - Function, String> unknownErrorMessageSource) { - type = Definition.ObjectClassTodefClass(type); + private static Class definitionTypeForClass(PainlessLookup painlessLookup, Class type, + Function, String> unknownErrorMessageSource) { + type = PainlessLookup.ObjectClassTodefClass(type); Class componentType = type; while (componentType.isArray()) { componentType = componentType.getComponentType(); } - if (definition.getPainlessStructFromJavaClass(componentType) == null) { + if (painlessLookup.getPainlessStructFromJavaClass(componentType) == null) { throw new IllegalArgumentException(unknownErrorMessageSource.apply(componentType)); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java index 18d7d94492e67..db3aeff0483f6 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/WriterConstants.java @@ -20,6 +20,7 @@ package org.elasticsearch.painless; import org.elasticsearch.painless.api.Augmentation; +import org.elasticsearch.painless.lookup.PainlessLookup; import org.elasticsearch.script.ScriptException; import org.objectweb.asm.Handle; import org.objectweb.asm.Opcodes; @@ -74,12 +75,12 @@ public final class WriterConstants { public static final Type STACK_OVERFLOW_ERROR_TYPE = Type.getType(StackOverflowError.class); public static final Type EXCEPTION_TYPE = Type.getType(Exception.class); public static final Type PAINLESS_EXPLAIN_ERROR_TYPE = Type.getType(PainlessExplainError.class); - public static final Method PAINLESS_EXPLAIN_ERROR_GET_HEADERS_METHOD = getAsmMethod(Map.class, "getHeaders", Definition.class); + public static final Method PAINLESS_EXPLAIN_ERROR_GET_HEADERS_METHOD = getAsmMethod(Map.class, "getHeaders", PainlessLookup.class); public static final Type OBJECT_TYPE = Type.getType(Object.class); public static final Type BITSET_TYPE = Type.getType(BitSet.class); - public static final Type DEFINITION_TYPE = Type.getType(Definition.class); + public static final Type DEFINITION_TYPE = Type.getType(PainlessLookup.class); public static final Type COLLECTIONS_TYPE = Type.getType(Collections.class); public static final Method EMPTY_MAP_METHOD = getAsmMethod(Map.class, "emptyMap"); @@ -103,10 +104,10 @@ public final class WriterConstants { public static final Type AUGMENTATION_TYPE = Type.getType(Augmentation.class); /** - * A Method instance for {@linkplain Pattern#compile}. This isn't available from Definition because we intentionally don't add it there - * so that the script can't create regexes without this syntax. Essentially, our static regex syntax has a monopoly on building regexes - * because it can do it statically. This is both faster and prevents the script from doing something super slow like building a regex - * per time it is run. + * A Method instance for {@linkplain Pattern#compile}. This isn't available from PainlessLookup because we intentionally don't add it + * there so that the script can't create regexes without this syntax. Essentially, our static regex syntax has a monopoly on building + * regexes because it can do it statically. This is both faster and prevents the script from doing something super slow like building a + * regex per time it is run. */ public static final Method PATTERN_COMPILE = getAsmMethod(Pattern.class, "compile", String.class, int.class); public static final Method PATTERN_MATCHER = getAsmMethod(Matcher.class, "matcher", CharSequence.class); @@ -118,7 +119,7 @@ public final class WriterConstants { static final Handle DEF_BOOTSTRAP_HANDLE = new Handle(Opcodes.H_INVOKESTATIC, CLASS_TYPE.getInternalName(), "$bootstrapDef", DEF_BOOTSTRAP_METHOD.getDescriptor(), false); public static final Type DEF_BOOTSTRAP_DELEGATE_TYPE = Type.getType(DefBootstrap.class); - public static final Method DEF_BOOTSTRAP_DELEGATE_METHOD = getAsmMethod(CallSite.class, "bootstrap", Definition.class, + public static final Method DEF_BOOTSTRAP_DELEGATE_METHOD = getAsmMethod(CallSite.class, "bootstrap", PainlessLookup.class, MethodHandles.Lookup.class, String.class, MethodType.class, int.class, int.class, Object[].class); public static final Type DEF_UTIL_TYPE = Type.getType(Def.class); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedPainlessLexer.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedPainlessLexer.java index add3aaabe51e0..e6c5da057980f 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedPainlessLexer.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/EnhancedPainlessLexer.java @@ -23,7 +23,7 @@ import org.antlr.v4.runtime.LexerNoViableAltException; import org.antlr.v4.runtime.Token; import org.antlr.v4.runtime.misc.Interval; -import org.elasticsearch.painless.Definition; +import org.elasticsearch.painless.lookup.PainlessLookup; import org.elasticsearch.painless.Location; /** @@ -39,14 +39,14 @@ */ final class EnhancedPainlessLexer extends PainlessLexer { private final String sourceName; - private final Definition definition; + private final PainlessLookup painlessLookup; private Token current = null; - EnhancedPainlessLexer(CharStream charStream, String sourceName, Definition definition) { + EnhancedPainlessLexer(CharStream charStream, String sourceName, PainlessLookup painlessLookup) { super(charStream); this.sourceName = sourceName; - this.definition = definition; + this.painlessLookup = painlessLookup; } @Override @@ -75,7 +75,7 @@ public void recover(final LexerNoViableAltException lnvae) { @Override protected boolean isType(String name) { - return definition.isSimplePainlessType(name); + return painlessLookup.isSimplePainlessType(name); } @Override diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java index a481c99a99d12..e2742ffb9936d 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/antlr/Walker.java @@ -29,7 +29,7 @@ import org.antlr.v4.runtime.atn.PredictionMode; import org.antlr.v4.runtime.tree.TerminalNode; import org.elasticsearch.painless.CompilerSettings; -import org.elasticsearch.painless.Definition; +import org.elasticsearch.painless.lookup.PainlessLookup; import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.Operation; @@ -174,9 +174,9 @@ public final class Walker extends PainlessParserBaseVisitor { public static SSource buildPainlessTree(ScriptClassInfo mainMethod, MainMethodReserved reserved, String sourceName, - String sourceText, CompilerSettings settings, Definition definition, + String sourceText, CompilerSettings settings, PainlessLookup painlessLookup, Printer debugStream) { - return new Walker(mainMethod, reserved, sourceName, sourceText, settings, definition, debugStream).source; + return new Walker(mainMethod, reserved, sourceName, sourceText, settings, painlessLookup, debugStream).source; } private final ScriptClassInfo scriptClassInfo; @@ -185,14 +185,14 @@ public static SSource buildPainlessTree(ScriptClassInfo mainMethod, MainMethodRe private final Printer debugStream; private final String sourceName; private final String sourceText; - private final Definition definition; + private final PainlessLookup painlessLookup; private final Deque reserved = new ArrayDeque<>(); private final Globals globals; private int syntheticCounter = 0; private Walker(ScriptClassInfo scriptClassInfo, MainMethodReserved reserved, String sourceName, String sourceText, - CompilerSettings settings, Definition definition, Printer debugStream) { + CompilerSettings settings, PainlessLookup painlessLookup, Printer debugStream) { this.scriptClassInfo = scriptClassInfo; this.reserved.push(reserved); this.debugStream = debugStream; @@ -200,13 +200,13 @@ private Walker(ScriptClassInfo scriptClassInfo, MainMethodReserved reserved, Str this.sourceName = Location.computeSourceName(sourceName); this.sourceText = sourceText; this.globals = new Globals(new BitSet(sourceText.length())); - this.definition = definition; + this.painlessLookup = painlessLookup; this.source = (SSource)visit(buildAntlrTree(sourceText)); } private SourceContext buildAntlrTree(String source) { ANTLRInputStream stream = new ANTLRInputStream(source); - PainlessLexer lexer = new EnhancedPainlessLexer(stream, sourceName, definition); + PainlessLexer lexer = new EnhancedPainlessLexer(stream, sourceName, painlessLookup); PainlessParser parser = new PainlessParser(new CommonTokenStream(lexer)); ParserErrorStrategy strategy = new ParserErrorStrategy(sourceName); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessCast.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessCast.java new file mode 100644 index 0000000000000..2440fb45d4dfb --- /dev/null +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessCast.java @@ -0,0 +1,67 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.painless.lookup; + +public class PainlessCast { + + /** Create a standard cast with no boxing/unboxing. */ + public static PainlessCast standard(Class from, Class to, boolean explicit) { + return new PainlessCast(from, to, explicit, null, null, null, null); + } + + /** Create a cast where the from type will be unboxed, and then the cast will be performed. */ + public static PainlessCast unboxFrom(Class from, Class to, boolean explicit, Class unboxFrom) { + return new PainlessCast(from, to, explicit, unboxFrom, null, null, null); + } + + /** Create a cast where the to type will be unboxed, and then the cast will be performed. */ + public static PainlessCast unboxTo(Class from, Class to, boolean explicit, Class unboxTo) { + return new PainlessCast(from, to, explicit, null, unboxTo, null, null); + } + + /** Create a cast where the from type will be boxed, and then the cast will be performed. */ + public static PainlessCast boxFrom(Class from, Class to, boolean explicit, Class boxFrom) { + return new PainlessCast(from, to, explicit, null, null, boxFrom, null); + } + + /** Create a cast where the to type will be boxed, and then the cast will be performed. */ + public static PainlessCast boxTo(Class from, Class to, boolean explicit, Class boxTo) { + return new PainlessCast(from, to, explicit, null, null, null, boxTo); + } + + public final Class from; + public final Class to; + public final boolean explicit; + public final Class unboxFrom; + public final Class unboxTo; + public final Class boxFrom; + public final Class boxTo; + + private PainlessCast(Class from, Class to, boolean explicit, + Class unboxFrom, Class unboxTo, Class boxFrom, Class boxTo) { + this.from = from; + this.to = to; + this.explicit = explicit; + this.unboxFrom = unboxFrom; + this.unboxTo = unboxTo; + this.boxFrom = boxFrom; + this.boxTo = boxTo; + } +} diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClass.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClass.java new file mode 100644 index 0000000000000..7d84899b00e58 --- /dev/null +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessClass.java @@ -0,0 +1,103 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.painless.lookup; + +import java.lang.invoke.MethodHandle; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public final class PainlessClass { + public final String name; + public final Class clazz; + public final org.objectweb.asm.Type type; + + public final Map constructors; + public final Map staticMethods; + public final Map methods; + + public final Map staticMembers; + public final Map members; + + public final Map getters; + public final Map setters; + + public final PainlessMethod functionalMethod; + + PainlessClass(String name, Class clazz, org.objectweb.asm.Type type) { + this.name = name; + this.clazz = clazz; + this.type = type; + + constructors = new HashMap<>(); + staticMethods = new HashMap<>(); + methods = new HashMap<>(); + + staticMembers = new HashMap<>(); + members = new HashMap<>(); + + getters = new HashMap<>(); + setters = new HashMap<>(); + + functionalMethod = null; + } + + private PainlessClass(PainlessClass struct, PainlessMethod functionalMethod) { + name = struct.name; + clazz = struct.clazz; + type = struct.type; + + constructors = Collections.unmodifiableMap(struct.constructors); + staticMethods = Collections.unmodifiableMap(struct.staticMethods); + methods = Collections.unmodifiableMap(struct.methods); + + staticMembers = Collections.unmodifiableMap(struct.staticMembers); + members = Collections.unmodifiableMap(struct.members); + + getters = Collections.unmodifiableMap(struct.getters); + setters = Collections.unmodifiableMap(struct.setters); + + this.functionalMethod = functionalMethod; + } + + public PainlessClass freeze(PainlessMethod functionalMethod) { + return new PainlessClass(this, functionalMethod); + } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + + if (object == null || getClass() != object.getClass()) { + return false; + } + + PainlessClass struct = (PainlessClass)object; + + return name.equals(struct.name); + } + + @Override + public int hashCode() { + return name.hashCode(); + } +} diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessField.java new file mode 100644 index 0000000000000..7c85bd269b461 --- /dev/null +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessField.java @@ -0,0 +1,43 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.painless.lookup; + +import java.lang.invoke.MethodHandle; + +public final class PainlessField { + public final String name; + public final PainlessClass owner; + public final Class clazz; + public final String javaName; + public final int modifiers; + public final MethodHandle getter; + public final MethodHandle setter; + + PainlessField(String name, String javaName, PainlessClass owner, Class clazz, int modifiers, + MethodHandle getter, MethodHandle setter) { + this.name = name; + this.javaName = javaName; + this.owner = owner; + this.clazz = clazz; + this.modifiers = modifiers; + this.getter = getter; + this.setter = setter; + } +} diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java similarity index 71% rename from modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java rename to modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java index 25145a44b5853..5833767fbd3d2 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/Definition.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessLookup.java @@ -17,15 +17,17 @@ * under the License. */ -package org.elasticsearch.painless; +package org.elasticsearch.painless.lookup; import org.elasticsearch.painless.spi.Whitelist; -import org.objectweb.asm.Opcodes; +import org.elasticsearch.painless.spi.WhitelistClass; +import org.elasticsearch.painless.spi.WhitelistConstructor; +import org.elasticsearch.painless.spi.WhitelistField; +import org.elasticsearch.painless.spi.WhitelistMethod; import org.objectweb.asm.Type; import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; -import java.lang.invoke.MethodType; import java.lang.reflect.Modifier; import java.util.ArrayList; import java.util.Arrays; @@ -34,7 +36,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Stack; import java.util.regex.Pattern; @@ -42,10 +43,10 @@ * The entire API for Painless. Also used as a whitelist for checking for legal * methods and fields during at both compile-time and runtime. */ -public final class Definition { +public final class PainlessLookup { - private static final Map methodCache = new HashMap<>(); - private static final Map fieldCache = new HashMap<>(); + private static final Map methodCache = new HashMap<>(); + private static final Map fieldCache = new HashMap<>(); private static final Pattern TYPE_NAME_PATTERN = Pattern.compile("^[_a-zA-Z][._a-zA-Z0-9]*$"); @@ -56,306 +57,6 @@ private def() { } } - public static class Method { - public final String name; - public final Struct owner; - public final Class augmentation; - public final Class rtn; - public final List> arguments; - public final org.objectweb.asm.commons.Method method; - public final int modifiers; - public final MethodHandle handle; - - public Method(String name, Struct owner, Class augmentation, Class rtn, List> arguments, - org.objectweb.asm.commons.Method method, int modifiers, MethodHandle handle) { - this.name = name; - this.augmentation = augmentation; - this.owner = owner; - this.rtn = rtn; - this.arguments = Collections.unmodifiableList(arguments); - this.method = method; - this.modifiers = modifiers; - this.handle = handle; - } - - /** - * Returns MethodType for this method. - *

    - * This works even for user-defined Methods (where the MethodHandle is null). - */ - public MethodType getMethodType() { - // we have a methodhandle already (e.g. whitelisted class) - // just return its type - if (handle != null) { - return handle.type(); - } - // otherwise compute it - final Class params[]; - final Class returnValue; - if (augmentation != null) { - // static method disguised as virtual/interface method - params = new Class[1 + arguments.size()]; - params[0] = augmentation; - for (int i = 0; i < arguments.size(); i++) { - params[i + 1] = defClassToObjectClass(arguments.get(i)); - } - returnValue = defClassToObjectClass(rtn); - } else if (Modifier.isStatic(modifiers)) { - // static method: straightforward copy - params = new Class[arguments.size()]; - for (int i = 0; i < arguments.size(); i++) { - params[i] = defClassToObjectClass(arguments.get(i)); - } - returnValue = defClassToObjectClass(rtn); - } else if ("".equals(name)) { - // constructor: returns the owner class - params = new Class[arguments.size()]; - for (int i = 0; i < arguments.size(); i++) { - params[i] = defClassToObjectClass(arguments.get(i)); - } - returnValue = owner.clazz; - } else { - // virtual/interface method: add receiver class - params = new Class[1 + arguments.size()]; - params[0] = owner.clazz; - for (int i = 0; i < arguments.size(); i++) { - params[i + 1] = defClassToObjectClass(arguments.get(i)); - } - returnValue = defClassToObjectClass(rtn); - } - return MethodType.methodType(returnValue, params); - } - - public void write(MethodWriter writer) { - final org.objectweb.asm.Type type; - final Class clazz; - if (augmentation != null) { - assert java.lang.reflect.Modifier.isStatic(modifiers); - clazz = augmentation; - type = org.objectweb.asm.Type.getType(augmentation); - } else { - clazz = owner.clazz; - type = owner.type; - } - - if (java.lang.reflect.Modifier.isStatic(modifiers)) { - // invokeStatic assumes that the owner class is not an interface, so this is a - // special case for interfaces where the interface method boolean needs to be set to - // true to reference the appropriate class constant when calling a static interface - // method since java 8 did not check, but java 9 and 10 do - if (java.lang.reflect.Modifier.isInterface(clazz.getModifiers())) { - writer.visitMethodInsn(Opcodes.INVOKESTATIC, - type.getInternalName(), name, getMethodType().toMethodDescriptorString(), true); - } else { - writer.invokeStatic(type, method); - } - } else if (java.lang.reflect.Modifier.isInterface(clazz.getModifiers())) { - writer.invokeInterface(type, method); - } else { - writer.invokeVirtual(type, method); - } - } - } - - public static final class Field { - public final String name; - public final Struct owner; - public final Class clazz; - public final String javaName; - public final int modifiers; - private final MethodHandle getter; - private final MethodHandle setter; - - private Field(String name, String javaName, Struct owner, Class clazz, int modifiers, MethodHandle getter, MethodHandle setter) { - this.name = name; - this.javaName = javaName; - this.owner = owner; - this.clazz = clazz; - this.modifiers = modifiers; - this.getter = getter; - this.setter = setter; - } - } - - // TODO: instead of hashing on this, we could have a 'next' pointer in Method itself, but it would make code more complex - // please do *NOT* under any circumstances change this to be the crappy Tuple from elasticsearch! - /** - * Key for looking up a method. - *

    - * Methods are keyed on both name and arity, and can be overloaded once per arity. - * This allows signatures such as {@code String.indexOf(String) vs String.indexOf(String, int)}. - *

    - * It is less flexible than full signature overloading where types can differ too, but - * better than just the name, and overloading types adds complexity to users, too. - */ - public static final class MethodKey { - public final String name; - public final int arity; - - /** - * Create a new lookup key - * @param name name of the method - * @param arity number of parameters - */ - public MethodKey(String name, int arity) { - this.name = Objects.requireNonNull(name); - this.arity = arity; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + arity; - result = prime * result + name.hashCode(); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null) return false; - if (getClass() != obj.getClass()) return false; - MethodKey other = (MethodKey) obj; - if (arity != other.arity) return false; - if (!name.equals(other.name)) return false; - return true; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append(name); - sb.append('/'); - sb.append(arity); - return sb.toString(); - } - } - - public static final class Struct { - public final String name; - public final Class clazz; - public final org.objectweb.asm.Type type; - - public final Map constructors; - public final Map staticMethods; - public final Map methods; - - public final Map staticMembers; - public final Map members; - - public final Map getters; - public final Map setters; - - public final Method functionalMethod; - - private Struct(String name, Class clazz, org.objectweb.asm.Type type) { - this.name = name; - this.clazz = clazz; - this.type = type; - - constructors = new HashMap<>(); - staticMethods = new HashMap<>(); - methods = new HashMap<>(); - - staticMembers = new HashMap<>(); - members = new HashMap<>(); - - getters = new HashMap<>(); - setters = new HashMap<>(); - - functionalMethod = null; - } - - private Struct(Struct struct, Method functionalMethod) { - name = struct.name; - clazz = struct.clazz; - type = struct.type; - - constructors = Collections.unmodifiableMap(struct.constructors); - staticMethods = Collections.unmodifiableMap(struct.staticMethods); - methods = Collections.unmodifiableMap(struct.methods); - - staticMembers = Collections.unmodifiableMap(struct.staticMembers); - members = Collections.unmodifiableMap(struct.members); - - getters = Collections.unmodifiableMap(struct.getters); - setters = Collections.unmodifiableMap(struct.setters); - - this.functionalMethod = functionalMethod; - } - - private Struct freeze(Method functionalMethod) { - return new Struct(this, functionalMethod); - } - - @Override - public boolean equals(Object object) { - if (this == object) { - return true; - } - - if (object == null || getClass() != object.getClass()) { - return false; - } - - Struct struct = (Struct)object; - - return name.equals(struct.name); - } - - @Override - public int hashCode() { - return name.hashCode(); - } - } - - public static class Cast { - - /** Create a standard cast with no boxing/unboxing. */ - public static Cast standard(Class from, Class to, boolean explicit) { - return new Cast(from, to, explicit, null, null, null, null); - } - - /** Create a cast where the from type will be unboxed, and then the cast will be performed. */ - public static Cast unboxFrom(Class from, Class to, boolean explicit, Class unboxFrom) { - return new Cast(from, to, explicit, unboxFrom, null, null, null); - } - - /** Create a cast where the to type will be unboxed, and then the cast will be performed. */ - public static Cast unboxTo(Class from, Class to, boolean explicit, Class unboxTo) { - return new Cast(from, to, explicit, null, unboxTo, null, null); - } - - /** Create a cast where the from type will be boxed, and then the cast will be performed. */ - public static Cast boxFrom(Class from, Class to, boolean explicit, Class boxFrom) { - return new Cast(from, to, explicit, null, null, boxFrom, null); - } - - /** Create a cast where the to type will be boxed, and then the cast will be performed. */ - public static Cast boxTo(Class from, Class to, boolean explicit, Class boxTo) { - return new Cast(from, to, explicit, null, null, null, boxTo); - } - - public final Class from; - public final Class to; - public final boolean explicit; - public final Class unboxFrom; - public final Class unboxTo; - public final Class boxFrom; - public final Class boxTo; - - private Cast(Class from, Class to, boolean explicit, Class unboxFrom, Class unboxTo, Class boxFrom, Class boxTo) { - this.from = from; - this.to = to; - this.explicit = explicit; - this.unboxFrom = unboxFrom; - this.unboxTo = unboxTo; - this.boxFrom = boxFrom; - this.boxTo = boxTo; - } - } - public static Class getBoxedType(Class clazz) { if (clazz == boolean.class) { return Boolean.class; @@ -520,29 +221,29 @@ private static String buildFieldCacheKey(String structName, String fieldName, St return structName + fieldName + typeName; } - public Collection getStructs() { + public Collection getStructs() { return javaClassesToPainlessStructs.values(); } private final Map> painlessTypesToJavaClasses; - private final Map, Struct> javaClassesToPainlessStructs; + private final Map, PainlessClass> javaClassesToPainlessStructs; - public Definition(List whitelists) { + public PainlessLookup(List whitelists) { painlessTypesToJavaClasses = new HashMap<>(); javaClassesToPainlessStructs = new HashMap<>(); String origin = null; painlessTypesToJavaClasses.put("def", def.class); - javaClassesToPainlessStructs.put(def.class, new Struct("def", Object.class, Type.getType(Object.class))); + javaClassesToPainlessStructs.put(def.class, new PainlessClass("def", Object.class, Type.getType(Object.class))); try { // first iteration collects all the Painless type names that // are used for validation during the second iteration for (Whitelist whitelist : whitelists) { - for (Whitelist.Struct whitelistStruct : whitelist.whitelistStructs) { + for (WhitelistClass whitelistStruct : whitelist.whitelistStructs) { String painlessTypeName = whitelistStruct.javaClassName.replace('$', '.'); - Struct painlessStruct = javaClassesToPainlessStructs.get(painlessTypesToJavaClasses.get(painlessTypeName)); + PainlessClass painlessStruct = javaClassesToPainlessStructs.get(painlessTypesToJavaClasses.get(painlessTypeName)); if (painlessStruct != null && painlessStruct.clazz.getName().equals(whitelistStruct.javaClassName) == false) { throw new IllegalArgumentException("struct [" + painlessStruct.name + "] cannot represent multiple classes " + @@ -561,20 +262,20 @@ public Definition(List whitelists) { // be available in Painless along with validating they exist and all their types have // been white-listed during the first iteration for (Whitelist whitelist : whitelists) { - for (Whitelist.Struct whitelistStruct : whitelist.whitelistStructs) { + for (WhitelistClass whitelistStruct : whitelist.whitelistStructs) { String painlessTypeName = whitelistStruct.javaClassName.replace('$', '.'); - for (Whitelist.Constructor whitelistConstructor : whitelistStruct.whitelistConstructors) { + for (WhitelistConstructor whitelistConstructor : whitelistStruct.whitelistConstructors) { origin = whitelistConstructor.origin; addConstructor(painlessTypeName, whitelistConstructor); } - for (Whitelist.Method whitelistMethod : whitelistStruct.whitelistMethods) { + for (WhitelistMethod whitelistMethod : whitelistStruct.whitelistMethods) { origin = whitelistMethod.origin; addMethod(whitelist.javaClassLoader, painlessTypeName, whitelistMethod); } - for (Whitelist.Field whitelistField : whitelistStruct.whitelistFields) { + for (WhitelistField whitelistField : whitelistStruct.whitelistFields) { origin = whitelistField.origin; addField(painlessTypeName, whitelistField); } @@ -587,7 +288,7 @@ public Definition(List whitelists) { // goes through each Painless struct and determines the inheritance list, // and then adds all inherited types to the Painless struct's whitelist for (Class javaClass : javaClassesToPainlessStructs.keySet()) { - Struct painlessStruct = javaClassesToPainlessStructs.get(javaClass); + PainlessClass painlessStruct = javaClassesToPainlessStructs.get(javaClass); List painlessSuperStructs = new ArrayList<>(); Class javaSuperClass = painlessStruct.clazz.getSuperclass(); @@ -598,7 +299,7 @@ public Definition(List whitelists) { // adds super classes to the inheritance list if (javaSuperClass != null && javaSuperClass.isInterface() == false) { while (javaSuperClass != null) { - Struct painlessSuperStruct = javaClassesToPainlessStructs.get(javaSuperClass); + PainlessClass painlessSuperStruct = javaClassesToPainlessStructs.get(javaSuperClass); if (painlessSuperStruct != null) { painlessSuperStructs.add(painlessSuperStruct.name); @@ -614,7 +315,7 @@ public Definition(List whitelists) { Class javaInterfaceLookup = javaInteraceLookups.pop(); for (Class javaSuperInterface : javaInterfaceLookup.getInterfaces()) { - Struct painlessInterfaceStruct = javaClassesToPainlessStructs.get(javaSuperInterface); + PainlessClass painlessInterfaceStruct = javaClassesToPainlessStructs.get(javaSuperInterface); if (painlessInterfaceStruct != null) { String painlessInterfaceStructName = painlessInterfaceStruct.name; @@ -635,7 +336,7 @@ public Definition(List whitelists) { // copies methods and fields from Object into interface types if (painlessStruct.clazz.isInterface() || (def.class.getSimpleName()).equals(painlessStruct.name)) { - Struct painlessObjectStruct = javaClassesToPainlessStructs.get(Object.class); + PainlessClass painlessObjectStruct = javaClassesToPainlessStructs.get(Object.class); if (painlessObjectStruct != null) { copyStruct(painlessStruct.name, Collections.singletonList(painlessObjectStruct.name)); @@ -644,17 +345,17 @@ public Definition(List whitelists) { } // precompute runtime classes - for (Struct painlessStruct : javaClassesToPainlessStructs.values()) { + for (PainlessClass painlessStruct : javaClassesToPainlessStructs.values()) { addRuntimeClass(painlessStruct); } // copy all structs to make them unmodifiable for outside users: - for (Map.Entry,Struct> entry : javaClassesToPainlessStructs.entrySet()) { + for (Map.Entry,PainlessClass> entry : javaClassesToPainlessStructs.entrySet()) { entry.setValue(entry.getValue().freeze(computeFunctionalInterfaceMethod(entry.getValue()))); } } - private void addStruct(ClassLoader whitelistClassLoader, Whitelist.Struct whitelistStruct) { + private void addStruct(ClassLoader whitelistClassLoader, WhitelistClass whitelistStruct) { String painlessTypeName = whitelistStruct.javaClassName.replace('$', '.'); String importedPainlessTypeName = painlessTypeName; @@ -688,10 +389,10 @@ private void addStruct(ClassLoader whitelistClassLoader, Whitelist.Struct whitel } } - Struct existingStruct = javaClassesToPainlessStructs.get(javaClass); + PainlessClass existingStruct = javaClassesToPainlessStructs.get(javaClass); if (existingStruct == null) { - Struct struct = new Struct(painlessTypeName, javaClass, org.objectweb.asm.Type.getType(javaClass)); + PainlessClass struct = new PainlessClass(painlessTypeName, javaClass, org.objectweb.asm.Type.getType(javaClass)); painlessTypesToJavaClasses.put(painlessTypeName, javaClass); javaClassesToPainlessStructs.put(javaClass, struct); } else if (existingStruct.clazz.equals(javaClass) == false) { @@ -725,8 +426,8 @@ private void addStruct(ClassLoader whitelistClassLoader, Whitelist.Struct whitel } } - private void addConstructor(String ownerStructName, Whitelist.Constructor whitelistConstructor) { - Struct ownerStruct = javaClassesToPainlessStructs.get(painlessTypesToJavaClasses.get(ownerStructName)); + private void addConstructor(String ownerStructName, WhitelistConstructor whitelistConstructor) { + PainlessClass ownerStruct = javaClassesToPainlessStructs.get(painlessTypesToJavaClasses.get(ownerStructName)); if (ownerStruct == null) { throw new IllegalArgumentException("owner struct [" + ownerStructName + "] not defined for constructor with " + @@ -760,8 +461,8 @@ private void addConstructor(String ownerStructName, Whitelist.Constructor whitel " with constructor parameters " + whitelistConstructor.painlessParameterTypeNames, exception); } - MethodKey painlessMethodKey = new MethodKey("", whitelistConstructor.painlessParameterTypeNames.size()); - Method painlessConstructor = ownerStruct.constructors.get(painlessMethodKey); + PainlessMethodKey painlessMethodKey = new PainlessMethodKey("", whitelistConstructor.painlessParameterTypeNames.size()); + PainlessMethod painlessConstructor = ownerStruct.constructors.get(painlessMethodKey); if (painlessConstructor == null) { org.objectweb.asm.commons.Method asmConstructor = org.objectweb.asm.commons.Method.getMethod(javaConstructor); @@ -775,7 +476,7 @@ private void addConstructor(String ownerStructName, Whitelist.Constructor whitel } painlessConstructor = methodCache.computeIfAbsent(buildMethodCacheKey(ownerStruct.name, "", painlessParametersTypes), - key -> new Method("", ownerStruct, null, void.class, painlessParametersTypes, + key -> new PainlessMethod("", ownerStruct, null, void.class, painlessParametersTypes, asmConstructor, javaConstructor.getModifiers(), javaHandle)); ownerStruct.constructors.put(painlessMethodKey, painlessConstructor); } else if (painlessConstructor.arguments.equals(painlessParametersTypes) == false){ @@ -785,8 +486,8 @@ private void addConstructor(String ownerStructName, Whitelist.Constructor whitel } } - private void addMethod(ClassLoader whitelistClassLoader, String ownerStructName, Whitelist.Method whitelistMethod) { - Struct ownerStruct = javaClassesToPainlessStructs.get(painlessTypesToJavaClasses.get(ownerStructName)); + private void addMethod(ClassLoader whitelistClassLoader, String ownerStructName, WhitelistMethod whitelistMethod) { + PainlessClass ownerStruct = javaClassesToPainlessStructs.get(painlessTypesToJavaClasses.get(ownerStructName)); if (ownerStruct == null) { throw new IllegalArgumentException("owner struct [" + ownerStructName + "] not defined for method with " + @@ -864,10 +565,11 @@ private void addMethod(ClassLoader whitelistClassLoader, String ownerStructName, "and parameters " + whitelistMethod.painlessParameterTypeNames); } - MethodKey painlessMethodKey = new MethodKey(whitelistMethod.javaMethodName, whitelistMethod.painlessParameterTypeNames.size()); + PainlessMethodKey painlessMethodKey = + new PainlessMethodKey(whitelistMethod.javaMethodName, whitelistMethod.painlessParameterTypeNames.size()); if (javaAugmentedClass == null && Modifier.isStatic(javaMethod.getModifiers())) { - Method painlessMethod = ownerStruct.staticMethods.get(painlessMethodKey); + PainlessMethod painlessMethod = ownerStruct.staticMethods.get(painlessMethodKey); if (painlessMethod == null) { org.objectweb.asm.commons.Method asmMethod = org.objectweb.asm.commons.Method.getMethod(javaMethod); @@ -882,8 +584,8 @@ private void addMethod(ClassLoader whitelistClassLoader, String ownerStructName, painlessMethod = methodCache.computeIfAbsent( buildMethodCacheKey(ownerStruct.name, whitelistMethod.javaMethodName, painlessParametersTypes), - key -> new Method(whitelistMethod.javaMethodName, ownerStruct, null, painlessReturnClass, painlessParametersTypes, - asmMethod, javaMethod.getModifiers(), javaMethodHandle)); + key -> new PainlessMethod(whitelistMethod.javaMethodName, ownerStruct, null, painlessReturnClass, + painlessParametersTypes, asmMethod, javaMethod.getModifiers(), javaMethodHandle)); ownerStruct.staticMethods.put(painlessMethodKey, painlessMethod); } else if ((painlessMethod.name.equals(whitelistMethod.javaMethodName) && painlessMethod.rtn == painlessReturnClass && painlessMethod.arguments.equals(painlessParametersTypes)) == false) { @@ -893,7 +595,7 @@ private void addMethod(ClassLoader whitelistClassLoader, String ownerStructName, "and parameters " + painlessParametersTypes + " and " + painlessMethod.arguments); } } else { - Method painlessMethod = ownerStruct.methods.get(painlessMethodKey); + PainlessMethod painlessMethod = ownerStruct.methods.get(painlessMethodKey); if (painlessMethod == null) { org.objectweb.asm.commons.Method asmMethod = org.objectweb.asm.commons.Method.getMethod(javaMethod); @@ -908,7 +610,7 @@ private void addMethod(ClassLoader whitelistClassLoader, String ownerStructName, painlessMethod = methodCache.computeIfAbsent( buildMethodCacheKey(ownerStruct.name, whitelistMethod.javaMethodName, painlessParametersTypes), - key -> new Method(whitelistMethod.javaMethodName, ownerStruct, javaAugmentedClass, painlessReturnClass, + key -> new PainlessMethod(whitelistMethod.javaMethodName, ownerStruct, javaAugmentedClass, painlessReturnClass, painlessParametersTypes, asmMethod, javaMethod.getModifiers(), javaMethodHandle)); ownerStruct.methods.put(painlessMethodKey, painlessMethod); } else if ((painlessMethod.name.equals(whitelistMethod.javaMethodName) && painlessMethod.rtn.equals(painlessReturnClass) && @@ -921,8 +623,8 @@ private void addMethod(ClassLoader whitelistClassLoader, String ownerStructName, } } - private void addField(String ownerStructName, Whitelist.Field whitelistField) { - Struct ownerStruct = javaClassesToPainlessStructs.get(painlessTypesToJavaClasses.get(ownerStructName)); + private void addField(String ownerStructName, WhitelistField whitelistField) { + PainlessClass ownerStruct = javaClassesToPainlessStructs.get(painlessTypesToJavaClasses.get(ownerStructName)); if (ownerStruct == null) { throw new IllegalArgumentException("owner struct [" + ownerStructName + "] not defined for method with " + @@ -958,12 +660,12 @@ private void addField(String ownerStructName, Whitelist.Field whitelistField) { "with owner struct [" + ownerStruct.name + "] is not final"); } - Field painlessField = ownerStruct.staticMembers.get(whitelistField.javaFieldName); + PainlessField painlessField = ownerStruct.staticMembers.get(whitelistField.javaFieldName); if (painlessField == null) { painlessField = fieldCache.computeIfAbsent( buildFieldCacheKey(ownerStruct.name, whitelistField.javaFieldName, painlessFieldClass.getName()), - key -> new Field(whitelistField.javaFieldName, javaField.getName(), + key -> new PainlessField(whitelistField.javaFieldName, javaField.getName(), ownerStruct, painlessFieldClass, javaField.getModifiers(), null, null)); ownerStruct.staticMembers.put(whitelistField.javaFieldName, painlessField); } else if (painlessField.clazz != painlessFieldClass) { @@ -987,12 +689,12 @@ private void addField(String ownerStructName, Whitelist.Field whitelistField) { " not found for class [" + ownerStruct.clazz.getName() + "]."); } - Field painlessField = ownerStruct.members.get(whitelistField.javaFieldName); + PainlessField painlessField = ownerStruct.members.get(whitelistField.javaFieldName); if (painlessField == null) { painlessField = fieldCache.computeIfAbsent( buildFieldCacheKey(ownerStruct.name, whitelistField.javaFieldName, painlessFieldClass.getName()), - key -> new Field(whitelistField.javaFieldName, javaField.getName(), + key -> new PainlessField(whitelistField.javaFieldName, javaField.getName(), ownerStruct, painlessFieldClass, javaField.getModifiers(), javaMethodHandleGetter, javaMethodHandleSetter)); ownerStruct.members.put(whitelistField.javaFieldName, painlessField); } else if (painlessField.clazz != painlessFieldClass) { @@ -1003,14 +705,14 @@ private void addField(String ownerStructName, Whitelist.Field whitelistField) { } private void copyStruct(String struct, List children) { - final Struct owner = javaClassesToPainlessStructs.get(painlessTypesToJavaClasses.get(struct)); + final PainlessClass owner = javaClassesToPainlessStructs.get(painlessTypesToJavaClasses.get(struct)); if (owner == null) { throw new IllegalArgumentException("Owner struct [" + struct + "] not defined for copy."); } for (int count = 0; count < children.size(); ++count) { - final Struct child = javaClassesToPainlessStructs.get(painlessTypesToJavaClasses.get(children.get(count))); + final PainlessClass child = javaClassesToPainlessStructs.get(painlessTypesToJavaClasses.get(children.get(count))); if (child == null) { throw new IllegalArgumentException("Child struct [" + children.get(count) + "]" + @@ -1022,9 +724,9 @@ private void copyStruct(String struct, List children) { " is not a super type of owner struct [" + owner.name + "] in copy."); } - for (Map.Entry kvPair : child.methods.entrySet()) { - MethodKey methodKey = kvPair.getKey(); - Method method = kvPair.getValue(); + for (Map.Entry kvPair : child.methods.entrySet()) { + PainlessMethodKey methodKey = kvPair.getKey(); + PainlessMethod method = kvPair.getValue(); if (owner.methods.get(methodKey) == null) { // TODO: some of these are no longer valid or outright don't work // TODO: since classes may not come from the Painless classloader @@ -1076,10 +778,10 @@ private void copyStruct(String struct, List children) { } } - for (Field field : child.members.values()) { + for (PainlessField field : child.members.values()) { if (owner.members.get(field.name) == null) { owner.members.put(field.name, - new Field(field.name, field.javaName, owner, field.clazz, field.modifiers, field.getter, field.setter)); + new PainlessField(field.name, field.javaName, owner, field.clazz, field.modifiers, field.getter, field.setter)); } } } @@ -1088,11 +790,11 @@ private void copyStruct(String struct, List children) { /** * Precomputes a more efficient structure for dynamic method/field access. */ - private void addRuntimeClass(final Struct struct) { + private void addRuntimeClass(final PainlessClass struct) { // add all getters/setters - for (Map.Entry method : struct.methods.entrySet()) { + for (Map.Entry method : struct.methods.entrySet()) { String name = method.getKey().name; - Method m = method.getValue(); + PainlessMethod m = method.getValue(); if (m.arguments.size() == 0 && name.startsWith("get") && @@ -1124,14 +826,14 @@ private void addRuntimeClass(final Struct struct) { } // add all members - for (Map.Entry member : struct.members.entrySet()) { + for (Map.Entry member : struct.members.entrySet()) { struct.getters.put(member.getKey(), member.getValue().getter); struct.setters.put(member.getKey(), member.getValue().setter); } } /** computes the functional interface method for a class, or returns null */ - private Method computeFunctionalInterfaceMethod(Struct clazz) { + private PainlessMethod computeFunctionalInterfaceMethod(PainlessClass clazz) { if (!clazz.clazz.isInterface()) { return null; } @@ -1166,7 +868,7 @@ private Method computeFunctionalInterfaceMethod(Struct clazz) { } // inspect the one method found from the reflection API, it should match the whitelist! java.lang.reflect.Method oneMethod = methods.get(0); - Method painless = clazz.methods.get(new Definition.MethodKey(oneMethod.getName(), oneMethod.getParameterCount())); + PainlessMethod painless = clazz.methods.get(new PainlessMethodKey(oneMethod.getName(), oneMethod.getParameterCount())); if (painless == null || painless.method.equals(org.objectweb.asm.commons.Method.getMethod(oneMethod)) == false) { throw new IllegalArgumentException("Class: " + clazz.name + " is functional but the functional " + "method is not whitelisted!"); @@ -1178,7 +880,7 @@ public boolean isSimplePainlessType(String painlessType) { return painlessTypesToJavaClasses.containsKey(painlessType); } - public Struct getPainlessStructFromJavaClass(Class clazz) { + public PainlessClass getPainlessStructFromJavaClass(Class clazz) { return javaClassesToPainlessStructs.get(clazz); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethod.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethod.java new file mode 100644 index 0000000000000..e87e1d4bf38fc --- /dev/null +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethod.java @@ -0,0 +1,130 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.painless.lookup; + +import org.elasticsearch.painless.MethodWriter; +import org.objectweb.asm.Opcodes; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodType; +import java.lang.reflect.Modifier; +import java.util.Collections; +import java.util.List; + +public class PainlessMethod { + public final String name; + public final PainlessClass owner; + public final Class augmentation; + public final Class rtn; + public final List> arguments; + public final org.objectweb.asm.commons.Method method; + public final int modifiers; + public final MethodHandle handle; + + public PainlessMethod(String name, PainlessClass owner, Class augmentation, Class rtn, List> arguments, + org.objectweb.asm.commons.Method method, int modifiers, MethodHandle handle) { + this.name = name; + this.augmentation = augmentation; + this.owner = owner; + this.rtn = rtn; + this.arguments = Collections.unmodifiableList(arguments); + this.method = method; + this.modifiers = modifiers; + this.handle = handle; + } + + /** + * Returns MethodType for this method. + *

    + * This works even for user-defined Methods (where the MethodHandle is null). + */ + public MethodType getMethodType() { + // we have a methodhandle already (e.g. whitelisted class) + // just return its type + if (handle != null) { + return handle.type(); + } + // otherwise compute it + final Class params[]; + final Class returnValue; + if (augmentation != null) { + // static method disguised as virtual/interface method + params = new Class[1 + arguments.size()]; + params[0] = augmentation; + for (int i = 0; i < arguments.size(); i++) { + params[i + 1] = PainlessLookup.defClassToObjectClass(arguments.get(i)); + } + returnValue = PainlessLookup.defClassToObjectClass(rtn); + } else if (Modifier.isStatic(modifiers)) { + // static method: straightforward copy + params = new Class[arguments.size()]; + for (int i = 0; i < arguments.size(); i++) { + params[i] = PainlessLookup.defClassToObjectClass(arguments.get(i)); + } + returnValue = PainlessLookup.defClassToObjectClass(rtn); + } else if ("".equals(name)) { + // constructor: returns the owner class + params = new Class[arguments.size()]; + for (int i = 0; i < arguments.size(); i++) { + params[i] = PainlessLookup.defClassToObjectClass(arguments.get(i)); + } + returnValue = owner.clazz; + } else { + // virtual/interface method: add receiver class + params = new Class[1 + arguments.size()]; + params[0] = owner.clazz; + for (int i = 0; i < arguments.size(); i++) { + params[i + 1] = PainlessLookup.defClassToObjectClass(arguments.get(i)); + } + returnValue = PainlessLookup.defClassToObjectClass(rtn); + } + return MethodType.methodType(returnValue, params); + } + + public void write(MethodWriter writer) { + final org.objectweb.asm.Type type; + final Class clazz; + if (augmentation != null) { + assert Modifier.isStatic(modifiers); + clazz = augmentation; + type = org.objectweb.asm.Type.getType(augmentation); + } else { + clazz = owner.clazz; + type = owner.type; + } + + if (Modifier.isStatic(modifiers)) { + // invokeStatic assumes that the owner class is not an interface, so this is a + // special case for interfaces where the interface method boolean needs to be set to + // true to reference the appropriate class constant when calling a static interface + // method since java 8 did not check, but java 9 and 10 do + if (Modifier.isInterface(clazz.getModifiers())) { + writer.visitMethodInsn(Opcodes.INVOKESTATIC, + type.getInternalName(), name, getMethodType().toMethodDescriptorString(), true); + } else { + writer.invokeStatic(type, method); + } + } else if (Modifier.isInterface(clazz.getModifiers())) { + writer.invokeInterface(type, method); + } else { + writer.invokeVirtual(type, method); + } + } +} diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethodKey.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethodKey.java new file mode 100644 index 0000000000000..49413ab0c5fef --- /dev/null +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/lookup/PainlessMethodKey.java @@ -0,0 +1,75 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.painless.lookup; + +import java.util.Objects; + +/** + * Key for looking up a method. + *

    + * Methods are keyed on both name and arity, and can be overloaded once per arity. + * This allows signatures such as {@code String.indexOf(String) vs String.indexOf(String, int)}. + *

    + * It is less flexible than full signature overloading where types can differ too, but + * better than just the name, and overloading types adds complexity to users, too. + */ +public final class PainlessMethodKey { + public final String name; + public final int arity; + + /** + * Create a new lookup key + * @param name name of the method + * @param arity number of parameters + */ + public PainlessMethodKey(String name, int arity) { + this.name = Objects.requireNonNull(name); + this.arity = arity; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + arity; + result = prime * result + name.hashCode(); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (getClass() != obj.getClass()) return false; + PainlessMethodKey other = (PainlessMethodKey) obj; + if (arity != other.arity) return false; + if (!name.equals(other.name)) return false; + return true; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(name); + sb.append('/'); + sb.append(arity); + return sb.toString(); + } +} diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java index 550e9db9495b7..0dfcf4d409335 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/AExpression.java @@ -20,8 +20,8 @@ package org.elasticsearch.painless.node; import org.elasticsearch.painless.AnalyzerCaster; -import org.elasticsearch.painless.Definition; -import org.elasticsearch.painless.Definition.Cast; +import org.elasticsearch.painless.lookup.PainlessLookup; +import org.elasticsearch.painless.lookup.PainlessCast; import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Location; @@ -118,7 +118,7 @@ public abstract class AExpression extends ANode { * @return The new child node for the parent node calling this method. */ AExpression cast(Locals locals) { - Cast cast = AnalyzerCaster.getLegalCast(location, actual, expected, explicit, internal); + PainlessCast cast = AnalyzerCaster.getLegalCast(location, actual, expected, explicit, internal); if (cast == null) { if (constant == null || this instanceof EConstant) { @@ -157,7 +157,7 @@ AExpression cast(Locals locals) { return ecast; } else { - if (Definition.isConstantType(expected)) { + if (PainlessLookup.isConstantType(expected)) { // For the case where a cast is required, a constant is set, // and the constant can be immediately cast to the expected type. // An EConstant replaces this node with the constant cast appropriately diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java index dce04661dbca8..dda246b5f6cda 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EAssignment.java @@ -22,8 +22,8 @@ import org.elasticsearch.painless.AnalyzerCaster; import org.elasticsearch.painless.DefBootstrap; -import org.elasticsearch.painless.Definition.Cast; -import org.elasticsearch.painless.Definition.def; +import org.elasticsearch.painless.lookup.PainlessCast; +import org.elasticsearch.painless.lookup.PainlessLookup.def; import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Location; @@ -49,8 +49,8 @@ public final class EAssignment extends AExpression { private boolean cat = false; private Class promote = null; private Class shiftDistance; // for shifts, the RHS is promoted independently - private Cast there = null; - private Cast back = null; + private PainlessCast there = null; + private PainlessCast back = null; public EAssignment(Location location, AExpression lhs, AExpression rhs, boolean pre, boolean post, Operation operation) { super(location); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java index 00168549b783e..46fbeefd6f557 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EBinary.java @@ -21,8 +21,8 @@ import org.elasticsearch.painless.AnalyzerCaster; import org.elasticsearch.painless.DefBootstrap; -import org.elasticsearch.painless.Definition; -import org.elasticsearch.painless.Definition.def; +import org.elasticsearch.painless.lookup.PainlessLookup; +import org.elasticsearch.painless.lookup.PainlessLookup.def; import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Location; @@ -106,7 +106,7 @@ private void analyzeMul(Locals variables) { if (promote == null) { throw createError(new ClassCastException("Cannot apply multiply [*] to types " + - "[" + Definition.ClassToName(left.actual) + "] and [" + Definition.ClassToName(right.actual) + "].")); + "[" + PainlessLookup.ClassToName(left.actual) + "] and [" + PainlessLookup.ClassToName(right.actual) + "].")); } actual = promote; @@ -148,7 +148,7 @@ private void analyzeDiv(Locals variables) { if (promote == null) { throw createError(new ClassCastException("Cannot apply divide [/] to types " + - "[" + Definition.ClassToName(left.actual) + "] and [" + Definition.ClassToName(right.actual) + "].")); + "[" + PainlessLookup.ClassToName(left.actual) + "] and [" + PainlessLookup.ClassToName(right.actual) + "].")); } actual = promote; @@ -195,7 +195,7 @@ private void analyzeRem(Locals variables) { if (promote == null) { throw createError(new ClassCastException("Cannot apply remainder [%] to types " + - "[" + Definition.ClassToName(left.actual) + "] and [" + Definition.ClassToName(right.actual) + "].")); + "[" + PainlessLookup.ClassToName(left.actual) + "] and [" + PainlessLookup.ClassToName(right.actual) + "].")); } actual = promote; @@ -242,7 +242,7 @@ private void analyzeAdd(Locals variables) { if (promote == null) { throw createError(new ClassCastException("Cannot apply add [+] to types " + - "[" + Definition.ClassToName(left.actual) + "] and [" + Definition.ClassToName(right.actual) + "].")); + "[" + PainlessLookup.ClassToName(left.actual) + "] and [" + PainlessLookup.ClassToName(right.actual) + "].")); } actual = promote; @@ -300,7 +300,7 @@ private void analyzeSub(Locals variables) { if (promote == null) { throw createError(new ClassCastException("Cannot apply subtract [-] to types " + - "[" + Definition.ClassToName(left.actual) + "] and [" + Definition.ClassToName(right.actual) + "].")); + "[" + PainlessLookup.ClassToName(left.actual) + "] and [" + PainlessLookup.ClassToName(right.actual) + "].")); } actual = promote; @@ -358,7 +358,7 @@ private void analyzeLSH(Locals variables) { if (lhspromote == null || rhspromote == null) { throw createError(new ClassCastException("Cannot apply left shift [<<] to types " + - "[" + Definition.ClassToName(left.actual) + "] and [" + Definition.ClassToName(right.actual) + "].")); + "[" + PainlessLookup.ClassToName(left.actual) + "] and [" + PainlessLookup.ClassToName(right.actual) + "].")); } actual = promote = lhspromote; @@ -405,7 +405,7 @@ private void analyzeRSH(Locals variables) { if (lhspromote == null || rhspromote == null) { throw createError(new ClassCastException("Cannot apply right shift [>>] to types " + - "[" + Definition.ClassToName(left.actual) + "] and [" + Definition.ClassToName(right.actual) + "].")); + "[" + PainlessLookup.ClassToName(left.actual) + "] and [" + PainlessLookup.ClassToName(right.actual) + "].")); } actual = promote = lhspromote; @@ -455,7 +455,7 @@ private void analyzeUSH(Locals variables) { if (lhspromote == null || rhspromote == null) { throw createError(new ClassCastException("Cannot apply unsigned shift [>>>] to types " + - "[" + Definition.ClassToName(left.actual) + "] and [" + Definition.ClassToName(right.actual) + "].")); + "[" + PainlessLookup.ClassToName(left.actual) + "] and [" + PainlessLookup.ClassToName(right.actual) + "].")); } if (lhspromote == def.class || rhspromote == def.class) { @@ -498,7 +498,7 @@ private void analyzeBWAnd(Locals variables) { if (promote == null) { throw createError(new ClassCastException("Cannot apply and [&] to types " + - "[" + Definition.ClassToName(left.actual) + "] and [" + Definition.ClassToName(right.actual) + "].")); + "[" + PainlessLookup.ClassToName(left.actual) + "] and [" + PainlessLookup.ClassToName(right.actual) + "].")); } actual = promote; @@ -537,7 +537,7 @@ private void analyzeXor(Locals variables) { if (promote == null) { throw createError(new ClassCastException("Cannot apply xor [^] to types " + - "[" + Definition.ClassToName(left.actual) + "] and [" + Definition.ClassToName(right.actual) + "].")); + "[" + PainlessLookup.ClassToName(left.actual) + "] and [" + PainlessLookup.ClassToName(right.actual) + "].")); } actual = promote; @@ -577,7 +577,7 @@ private void analyzeBWOr(Locals variables) { if (promote == null) { throw createError(new ClassCastException("Cannot apply or [|] to types " + - "[" + Definition.ClassToName(left.actual) + "] and [" + Definition.ClassToName(right.actual) + "].")); + "[" + PainlessLookup.ClassToName(left.actual) + "] and [" + PainlessLookup.ClassToName(right.actual) + "].")); } actual = promote; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java index 4c977fa66e89a..ede1a2a6b9c36 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECallLocal.java @@ -19,8 +19,8 @@ package org.elasticsearch.painless.node; -import org.elasticsearch.painless.Definition.Method; -import org.elasticsearch.painless.Definition.MethodKey; +import org.elasticsearch.painless.lookup.PainlessMethod; +import org.elasticsearch.painless.lookup.PainlessMethodKey; import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Location; @@ -40,7 +40,7 @@ public final class ECallLocal extends AExpression { private final String name; private final List arguments; - private Method method = null; + private PainlessMethod method = null; public ECallLocal(Location location, String name, List arguments) { super(location); @@ -58,7 +58,7 @@ void extractVariables(Set variables) { @Override void analyze(Locals locals) { - MethodKey methodKey = new MethodKey(name, arguments.size()); + PainlessMethodKey methodKey = new PainlessMethodKey(name, arguments.size()); method = locals.getMethod(methodKey); if (method == null) { diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java index e6f2f7ebf91f9..a3e1b4bde6a86 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECapturingFunctionRef.java @@ -21,8 +21,8 @@ import org.elasticsearch.painless.AnalyzerCaster; import org.elasticsearch.painless.DefBootstrap; -import org.elasticsearch.painless.Definition; -import org.elasticsearch.painless.Definition.def; +import org.elasticsearch.painless.lookup.PainlessLookup; +import org.elasticsearch.painless.lookup.PainlessLookup.def; import org.elasticsearch.painless.FunctionRef; import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; @@ -69,7 +69,7 @@ void analyze(Locals locals) { defPointer = "D" + variable + "." + call + ",1"; } else { // typed implementation - defPointer = "S" + Definition.ClassToName(captured.clazz) + "." + call + ",1"; + defPointer = "S" + PainlessLookup.ClassToName(captured.clazz) + "." + call + ",1"; } actual = String.class; } else { @@ -77,7 +77,7 @@ void analyze(Locals locals) { // static case if (captured.clazz != def.class) { try { - ref = new FunctionRef(locals.getDefinition(), expected, Definition.ClassToName(captured.clazz), call, 1); + ref = new FunctionRef(locals.getPainlessLookup(), expected, PainlessLookup.ClassToName(captured.clazz), call, 1); // check casts between the interface method and the delegate method are legal for (int i = 0; i < ref.interfaceMethod.arguments.size(); ++i) { @@ -109,7 +109,7 @@ void write(MethodWriter writer, Globals globals) { // typed interface, dynamic implementation writer.visitVarInsn(MethodWriter.getType(captured.clazz).getOpcode(Opcodes.ILOAD), captured.getSlot()); Type methodType = Type.getMethodType(MethodWriter.getType(expected), MethodWriter.getType(captured.clazz)); - writer.invokeDefCall(call, methodType, DefBootstrap.REFERENCE, Definition.ClassToName(expected)); + writer.invokeDefCall(call, methodType, DefBootstrap.REFERENCE, PainlessLookup.ClassToName(expected)); } else { // typed interface, typed implementation writer.visitVarInsn(MethodWriter.getType(captured.clazz).getOpcode(Opcodes.ILOAD), captured.getSlot()); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java index 2f492fe6edb36..55a9dbf71c891 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ECast.java @@ -19,8 +19,8 @@ package org.elasticsearch.painless.node; -import org.elasticsearch.painless.Definition; -import org.elasticsearch.painless.Definition.Cast; +import org.elasticsearch.painless.lookup.PainlessLookup; +import org.elasticsearch.painless.lookup.PainlessCast; import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Location; @@ -35,9 +35,9 @@ final class ECast extends AExpression { private AExpression child; - private final Cast cast; + private final PainlessCast cast; - ECast(Location location, AExpression child, Cast cast) { + ECast(Location location, AExpression child, PainlessCast cast) { super(location); this.child = Objects.requireNonNull(child); @@ -63,6 +63,6 @@ void write(MethodWriter writer, Globals globals) { @Override public String toString() { - return singleLineToString(Definition.ClassToName(cast.to), child); + return singleLineToString(PainlessLookup.ClassToName(cast.to), child); } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java index 876229797ecb0..c0fccab8e8a8e 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EComp.java @@ -21,8 +21,8 @@ import org.elasticsearch.painless.AnalyzerCaster; import org.elasticsearch.painless.DefBootstrap; -import org.elasticsearch.painless.Definition; -import org.elasticsearch.painless.Definition.def; +import org.elasticsearch.painless.lookup.PainlessLookup; +import org.elasticsearch.painless.lookup.PainlessLookup.def; import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Location; @@ -93,7 +93,7 @@ private void analyzeEq(Locals variables) { if (promotedType == null) { throw createError(new ClassCastException("Cannot apply equals [==] to types " + - "[" + Definition.ClassToName(left.actual) + "] and [" + Definition.ClassToName(right.actual) + "].")); + "[" + PainlessLookup.ClassToName(left.actual) + "] and [" + PainlessLookup.ClassToName(right.actual) + "].")); } if (promotedType == def.class) { @@ -142,7 +142,7 @@ private void analyzeEqR(Locals variables) { if (promotedType == null) { throw createError(new ClassCastException("Cannot apply reference equals [===] to types " + - "[" + Definition.ClassToName(left.actual) + "] and [" + Definition.ClassToName(right.actual) + "].")); + "[" + PainlessLookup.ClassToName(left.actual) + "] and [" + PainlessLookup.ClassToName(right.actual) + "].")); } left.expected = promotedType; @@ -182,7 +182,7 @@ private void analyzeNE(Locals variables) { if (promotedType == null) { throw createError(new ClassCastException("Cannot apply not equals [!=] to types " + - "[" + Definition.ClassToName(left.actual) + "] and [" + Definition.ClassToName(right.actual) + "].")); + "[" + PainlessLookup.ClassToName(left.actual) + "] and [" + PainlessLookup.ClassToName(right.actual) + "].")); } if (promotedType == def.class) { @@ -231,7 +231,7 @@ private void analyzeNER(Locals variables) { if (promotedType == null) { throw createError(new ClassCastException("Cannot apply reference not equals [!==] to types " + - "[" + Definition.ClassToName(left.actual) + "] and [" + Definition.ClassToName(right.actual) + "].")); + "[" + PainlessLookup.ClassToName(left.actual) + "] and [" + PainlessLookup.ClassToName(right.actual) + "].")); } left.expected = promotedType; @@ -271,7 +271,7 @@ private void analyzeGTE(Locals variables) { if (promotedType == null) { throw createError(new ClassCastException("Cannot apply greater than or equals [>=] to types " + - "[" + Definition.ClassToName(left.actual) + "] and [" + Definition.ClassToName(right.actual) + "].")); + "[" + PainlessLookup.ClassToName(left.actual) + "] and [" + PainlessLookup.ClassToName(right.actual) + "].")); } if (promotedType == def.class) { @@ -310,7 +310,7 @@ private void analyzeGT(Locals variables) { if (promotedType == null) { throw createError(new ClassCastException("Cannot apply greater than [>] to types " + - "[" + Definition.ClassToName(left.actual) + "] and [" + Definition.ClassToName(right.actual) + "].")); + "[" + PainlessLookup.ClassToName(left.actual) + "] and [" + PainlessLookup.ClassToName(right.actual) + "].")); } if (promotedType == def.class) { @@ -349,7 +349,7 @@ private void analyzeLTE(Locals variables) { if (promotedType == null) { throw createError(new ClassCastException("Cannot apply less than or equals [<=] to types " + - "[" + Definition.ClassToName(left.actual) + "] and [" + Definition.ClassToName(right.actual) + "].")); + "[" + PainlessLookup.ClassToName(left.actual) + "] and [" + PainlessLookup.ClassToName(right.actual) + "].")); } if (promotedType == def.class) { @@ -388,7 +388,7 @@ private void analyzeLT(Locals variables) { if (promotedType == null) { throw createError(new ClassCastException("Cannot apply less than [>=] to types " + - "[" + Definition.ClassToName(left.actual) + "] and [" + Definition.ClassToName(right.actual) + "].")); + "[" + PainlessLookup.ClassToName(left.actual) + "] and [" + PainlessLookup.ClassToName(right.actual) + "].")); } if (promotedType == def.class) { diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java index 5a897e04a8d98..d19068f8fa6a2 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EExplicit.java @@ -50,7 +50,7 @@ void extractVariables(Set variables) { @Override void analyze(Locals locals) { try { - actual = locals.getDefinition().getJavaClassFromPainlessType(type); + actual = locals.getPainlessLookup().getJavaClassFromPainlessType(type); } catch (IllegalArgumentException exception) { throw createError(new IllegalArgumentException("Not a type [" + type + "].")); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java index 21bef9aa2ed5d..82c24e27c5d16 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EFunctionRef.java @@ -20,9 +20,9 @@ package org.elasticsearch.painless.node; import org.elasticsearch.painless.AnalyzerCaster; -import org.elasticsearch.painless.Definition; -import org.elasticsearch.painless.Definition.Method; -import org.elasticsearch.painless.Definition.MethodKey; +import org.elasticsearch.painless.lookup.PainlessLookup; +import org.elasticsearch.painless.lookup.PainlessMethod; +import org.elasticsearch.painless.lookup.PainlessMethodKey; import org.elasticsearch.painless.FunctionRef; import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; @@ -66,15 +66,15 @@ void analyze(Locals locals) { try { if ("this".equals(type)) { // user's own function - Method interfaceMethod = locals.getDefinition().getPainlessStructFromJavaClass(expected).functionalMethod; + PainlessMethod interfaceMethod = locals.getPainlessLookup().getPainlessStructFromJavaClass(expected).functionalMethod; if (interfaceMethod == null) { throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " + - "to [" + Definition.ClassToName(expected) + "], not a functional interface"); + "to [" + PainlessLookup.ClassToName(expected) + "], not a functional interface"); } - Method delegateMethod = locals.getMethod(new MethodKey(call, interfaceMethod.arguments.size())); + PainlessMethod delegateMethod = locals.getMethod(new PainlessMethodKey(call, interfaceMethod.arguments.size())); if (delegateMethod == null) { throw new IllegalArgumentException("Cannot convert function reference [" + type + "::" + call + "] " + - "to [" + Definition.ClassToName(expected) + "], function not found"); + "to [" + PainlessLookup.ClassToName(expected) + "], function not found"); } ref = new FunctionRef(expected, interfaceMethod, delegateMethod, 0); @@ -90,7 +90,7 @@ void analyze(Locals locals) { } } else { // whitelist lookup - ref = new FunctionRef(locals.getDefinition(), expected, type, call, 0); + ref = new FunctionRef(locals.getPainlessLookup(), expected, type, call, 0); } } catch (IllegalArgumentException e) { diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java index 5296d79e214ed..a3835cbc21372 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EInstanceof.java @@ -19,7 +19,7 @@ package org.elasticsearch.painless.node; -import org.elasticsearch.painless.Definition; +import org.elasticsearch.painless.lookup.PainlessLookup; import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Location; @@ -58,13 +58,13 @@ void analyze(Locals locals) { // ensure the specified type is part of the definition try { - clazz = locals.getDefinition().getJavaClassFromPainlessType(this.type); + clazz = locals.getPainlessLookup().getJavaClassFromPainlessType(this.type); } catch (IllegalArgumentException exception) { throw createError(new IllegalArgumentException("Not a type [" + this.type + "].")); } // map to wrapped type for primitive types - resolvedType = clazz.isPrimitive() ? Definition.getBoxedType(clazz) : Definition.defClassToObjectClass(clazz); + resolvedType = clazz.isPrimitive() ? PainlessLookup.getBoxedType(clazz) : PainlessLookup.defClassToObjectClass(clazz); // analyze and cast the expression expression.analyze(locals); @@ -75,7 +75,7 @@ void analyze(Locals locals) { primitiveExpression = expression.actual.isPrimitive(); // map to wrapped type for primitive types expressionType = expression.actual.isPrimitive() ? - Definition.getBoxedType(expression.actual) : Definition.defClassToObjectClass(clazz); + PainlessLookup.getBoxedType(expression.actual) : PainlessLookup.defClassToObjectClass(clazz); actual = boolean.class; } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java index e40d21ab110ab..a7b7a41fe051a 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ELambda.java @@ -20,9 +20,9 @@ package org.elasticsearch.painless.node; import org.elasticsearch.painless.AnalyzerCaster; -import org.elasticsearch.painless.Definition; -import org.elasticsearch.painless.Definition.Method; -import org.elasticsearch.painless.Definition.def; +import org.elasticsearch.painless.lookup.PainlessLookup; +import org.elasticsearch.painless.lookup.PainlessMethod; +import org.elasticsearch.painless.lookup.PainlessLookup.def; import org.elasticsearch.painless.FunctionRef; import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; @@ -103,7 +103,7 @@ void extractVariables(Set variables) { void analyze(Locals locals) { Class returnType; List actualParamTypeStrs; - Method interfaceMethod; + PainlessMethod interfaceMethod; // inspect the target first, set interface method if we know it. if (expected == null) { interfaceMethod = null; @@ -120,15 +120,15 @@ void analyze(Locals locals) { } } else { // we know the method statically, infer return type and any unknown/def types - interfaceMethod = locals.getDefinition().getPainlessStructFromJavaClass(expected).functionalMethod; + interfaceMethod = locals.getPainlessLookup().getPainlessStructFromJavaClass(expected).functionalMethod; if (interfaceMethod == null) { - throw createError(new IllegalArgumentException("Cannot pass lambda to [" + Definition.ClassToName(expected) + + throw createError(new IllegalArgumentException("Cannot pass lambda to [" + PainlessLookup.ClassToName(expected) + "], not a functional interface")); } // check arity before we manipulate parameters if (interfaceMethod.arguments.size() != paramTypeStrs.size()) throw new IllegalArgumentException("Incorrect number of parameters for [" + interfaceMethod.name + - "] in [" + Definition.ClassToName(expected) + "]"); + "] in [" + PainlessLookup.ClassToName(expected) + "]"); // for method invocation, its allowed to ignore the return value if (interfaceMethod.rtn == void.class) { returnType = def.class; @@ -140,7 +140,7 @@ void analyze(Locals locals) { for (int i = 0; i < paramTypeStrs.size(); i++) { String paramType = paramTypeStrs.get(i); if (paramType == null) { - actualParamTypeStrs.add(Definition.ClassToName(interfaceMethod.arguments.get(i))); + actualParamTypeStrs.add(PainlessLookup.ClassToName(interfaceMethod.arguments.get(i))); } else { actualParamTypeStrs.add(paramType); } @@ -162,16 +162,16 @@ void analyze(Locals locals) { List paramTypes = new ArrayList<>(captures.size() + actualParamTypeStrs.size()); List paramNames = new ArrayList<>(captures.size() + paramNameStrs.size()); for (Variable var : captures) { - paramTypes.add(Definition.ClassToName(var.clazz)); + paramTypes.add(PainlessLookup.ClassToName(var.clazz)); paramNames.add(var.name); } paramTypes.addAll(actualParamTypeStrs); paramNames.addAll(paramNameStrs); // desugar lambda body into a synthetic method - desugared = new SFunction(reserved, location, Definition.ClassToName(returnType), name, + desugared = new SFunction(reserved, location, PainlessLookup.ClassToName(returnType), name, paramTypes, paramNames, statements, true); - desugared.generateSignature(locals.getDefinition()); + desugared.generateSignature(locals.getPainlessLookup()); desugared.analyze(Locals.newLambdaScope(locals.getProgramScope(), returnType, desugared.parameters, captures.size(), reserved.getMaxLoopCounter())); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java index 05b10796cb4f9..518f1953525a6 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EListInit.java @@ -19,9 +19,9 @@ package org.elasticsearch.painless.node; -import org.elasticsearch.painless.Definition.Method; -import org.elasticsearch.painless.Definition.MethodKey; -import org.elasticsearch.painless.Definition.def; +import org.elasticsearch.painless.lookup.PainlessMethod; +import org.elasticsearch.painless.lookup.PainlessMethodKey; +import org.elasticsearch.painless.lookup.PainlessLookup.def; import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Location; @@ -37,8 +37,8 @@ public final class EListInit extends AExpression { private final List values; - private Method constructor = null; - private Method method = null; + private PainlessMethod constructor = null; + private PainlessMethod method = null; public EListInit(Location location, List values) { super(location); @@ -61,13 +61,14 @@ void analyze(Locals locals) { actual = ArrayList.class; - constructor = locals.getDefinition().getPainlessStructFromJavaClass(actual).constructors.get(new MethodKey("", 0)); + constructor = + locals.getPainlessLookup().getPainlessStructFromJavaClass(actual).constructors.get(new PainlessMethodKey("", 0)); if (constructor == null) { throw createError(new IllegalStateException("Illegal tree structure.")); } - method = locals.getDefinition().getPainlessStructFromJavaClass(actual).methods.get(new MethodKey("add", 1)); + method = locals.getPainlessLookup().getPainlessStructFromJavaClass(actual).methods.get(new PainlessMethodKey("add", 1)); if (method == null) { throw createError(new IllegalStateException("Illegal tree structure.")); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java index f5763042b8191..45158aedcf787 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EMapInit.java @@ -19,9 +19,9 @@ package org.elasticsearch.painless.node; -import org.elasticsearch.painless.Definition.Method; -import org.elasticsearch.painless.Definition.MethodKey; -import org.elasticsearch.painless.Definition.def; +import org.elasticsearch.painless.lookup.PainlessMethod; +import org.elasticsearch.painless.lookup.PainlessMethodKey; +import org.elasticsearch.painless.lookup.PainlessLookup.def; import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Location; @@ -38,8 +38,8 @@ public final class EMapInit extends AExpression { private final List keys; private final List values; - private Method constructor = null; - private Method method = null; + private PainlessMethod constructor = null; + private PainlessMethod method = null; public EMapInit(Location location, List keys, List values) { super(location); @@ -67,13 +67,14 @@ void analyze(Locals locals) { actual = HashMap.class; - constructor = locals.getDefinition().getPainlessStructFromJavaClass(actual).constructors.get(new MethodKey("", 0)); + constructor = + locals.getPainlessLookup().getPainlessStructFromJavaClass(actual).constructors.get(new PainlessMethodKey("", 0)); if (constructor == null) { throw createError(new IllegalStateException("Illegal tree structure.")); } - method = locals.getDefinition().getPainlessStructFromJavaClass(actual).methods.get(new MethodKey("put", 2)); + method = locals.getPainlessLookup().getPainlessStructFromJavaClass(actual).methods.get(new PainlessMethodKey("put", 2)); if (method == null) { throw createError(new IllegalStateException("Illegal tree structure.")); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java index 1a0a718ae7fc8..f9bd4cebc3fed 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewArray.java @@ -61,7 +61,7 @@ void analyze(Locals locals) { Class clazz; try { - clazz = locals.getDefinition().getJavaClassFromPainlessType(this.type); + clazz = locals.getPainlessLookup().getJavaClassFromPainlessType(this.type); } catch (IllegalArgumentException exception) { throw createError(new IllegalArgumentException("Not a type [" + this.type + "].")); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java index e3a926ef2244b..197e2fcb7fd42 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENewObj.java @@ -19,13 +19,13 @@ package org.elasticsearch.painless.node; -import org.elasticsearch.painless.Definition; -import org.elasticsearch.painless.Definition.Method; -import org.elasticsearch.painless.Definition.Struct; +import org.elasticsearch.painless.lookup.PainlessMethod; +import org.elasticsearch.painless.lookup.PainlessClass; import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.MethodWriter; +import org.elasticsearch.painless.lookup.PainlessMethodKey; import java.util.List; import java.util.Objects; @@ -39,7 +39,7 @@ public final class ENewObj extends AExpression { private final String type; private final List arguments; - private Method constructor; + private PainlessMethod constructor; public ENewObj(Location location, String type, List arguments) { super(location); @@ -58,13 +58,13 @@ void extractVariables(Set variables) { @Override void analyze(Locals locals) { try { - actual = locals.getDefinition().getJavaClassFromPainlessType(this.type); + actual = locals.getPainlessLookup().getJavaClassFromPainlessType(this.type); } catch (IllegalArgumentException exception) { throw createError(new IllegalArgumentException("Not a type [" + this.type + "].")); } - Struct struct = locals.getDefinition().getPainlessStructFromJavaClass(actual); - constructor = struct.constructors.get(new Definition.MethodKey("", arguments.size())); + PainlessClass struct = locals.getPainlessLookup().getPainlessStructFromJavaClass(actual); + constructor = struct.constructors.get(new PainlessMethodKey("", arguments.size())); if (constructor != null) { Class[] types = new Class[constructor.arguments.size()]; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java index ae682d5f7be3c..983819b6b2bf9 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/ENull.java @@ -19,7 +19,7 @@ package org.elasticsearch.painless.node; -import org.elasticsearch.painless.Definition; +import org.elasticsearch.painless.lookup.PainlessLookup; import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Location; @@ -53,7 +53,7 @@ void analyze(Locals locals) { if (expected != null) { if (expected.isPrimitive()) { throw createError(new IllegalArgumentException( - "Cannot cast null to a primitive type [" + Definition.ClassToName(expected) + "].")); + "Cannot cast null to a primitive type [" + PainlessLookup.ClassToName(expected) + "].")); } actual = expected; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EStatic.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EStatic.java index 5ebf30f5781cf..a556b3ad315c6 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EStatic.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EStatic.java @@ -48,7 +48,7 @@ void extractVariables(Set variables) { @Override void analyze(Locals locals) { try { - actual = locals.getDefinition().getJavaClassFromPainlessType(type); + actual = locals.getPainlessLookup().getJavaClassFromPainlessType(type); } catch (IllegalArgumentException exception) { throw createError(new IllegalArgumentException("Not a type [" + type + "].")); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java index a746ade26a15c..8e293556eac01 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/EUnary.java @@ -21,8 +21,8 @@ import org.elasticsearch.painless.AnalyzerCaster; import org.elasticsearch.painless.DefBootstrap; -import org.elasticsearch.painless.Definition; -import org.elasticsearch.painless.Definition.def; +import org.elasticsearch.painless.lookup.PainlessLookup; +import org.elasticsearch.painless.lookup.PainlessLookup.def; import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Location; @@ -93,7 +93,7 @@ void analyzeBWNot(Locals variables) { promote = AnalyzerCaster.promoteNumeric(child.actual, false); if (promote == null) { - throw createError(new ClassCastException("Cannot apply not [~] to type [" + Definition.ClassToName(child.actual) + "].")); + throw createError(new ClassCastException("Cannot apply not [~] to type [" + PainlessLookup.ClassToName(child.actual) + "].")); } child.expected = promote; @@ -122,7 +122,8 @@ void analyzerAdd(Locals variables) { promote = AnalyzerCaster.promoteNumeric(child.actual, true); if (promote == null) { - throw createError(new ClassCastException("Cannot apply positive [+] to type [" + Definition.ClassToName(child.actual) + "].")); + throw createError( + new ClassCastException("Cannot apply positive [+] to type [" + PainlessLookup.ClassToName(child.actual) + "].")); } child.expected = promote; @@ -155,7 +156,8 @@ void analyzerSub(Locals variables) { promote = AnalyzerCaster.promoteNumeric(child.actual, true); if (promote == null) { - throw createError(new ClassCastException("Cannot apply negative [-] to type [" + Definition.ClassToName(child.actual) + "].")); + throw createError( + new ClassCastException("Cannot apply negative [-] to type [" + PainlessLookup.ClassToName(child.actual) + "].")); } child.expected = promote; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java index 0e2ab70897fe5..ec7d0f6d7bb7a 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PBrace.java @@ -19,8 +19,8 @@ package org.elasticsearch.painless.node; -import org.elasticsearch.painless.Definition; -import org.elasticsearch.painless.Definition.def; +import org.elasticsearch.painless.lookup.PainlessLookup; +import org.elasticsearch.painless.lookup.PainlessLookup.def; import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Location; @@ -63,12 +63,12 @@ void analyze(Locals locals) { } else if (prefix.actual == def.class) { sub = new PSubDefArray(location, index); } else if (Map.class.isAssignableFrom(prefix.actual)) { - sub = new PSubMapShortcut(location, locals.getDefinition().getPainlessStructFromJavaClass(prefix.actual), index); + sub = new PSubMapShortcut(location, locals.getPainlessLookup().getPainlessStructFromJavaClass(prefix.actual), index); } else if (List.class.isAssignableFrom(prefix.actual)) { - sub = new PSubListShortcut(location, locals.getDefinition().getPainlessStructFromJavaClass(prefix.actual), index); + sub = new PSubListShortcut(location, locals.getPainlessLookup().getPainlessStructFromJavaClass(prefix.actual), index); } else { throw createError( - new IllegalArgumentException("Illegal array access on type [" + Definition.ClassToName(prefix.actual) + "].")); + new IllegalArgumentException("Illegal array access on type [" + PainlessLookup.ClassToName(prefix.actual) + "].")); } sub.write = write; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java index 6fff5a8e93f3e..12ff483248367 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PCallInvoke.java @@ -19,11 +19,11 @@ package org.elasticsearch.painless.node; -import org.elasticsearch.painless.Definition; -import org.elasticsearch.painless.Definition.Method; -import org.elasticsearch.painless.Definition.MethodKey; -import org.elasticsearch.painless.Definition.Struct; -import org.elasticsearch.painless.Definition.def; +import org.elasticsearch.painless.lookup.PainlessLookup; +import org.elasticsearch.painless.lookup.PainlessMethod; +import org.elasticsearch.painless.lookup.PainlessMethodKey; +import org.elasticsearch.painless.lookup.PainlessClass; +import org.elasticsearch.painless.lookup.PainlessLookup.def; import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Location; @@ -71,14 +71,14 @@ void analyze(Locals locals) { throw createError(new IllegalArgumentException("Illegal call [" + name + "] on array type.")); } - Struct struct = locals.getDefinition().getPainlessStructFromJavaClass(prefix.actual); + PainlessClass struct = locals.getPainlessLookup().getPainlessStructFromJavaClass(prefix.actual); if (prefix.actual.isPrimitive()) { - struct = locals.getDefinition().getPainlessStructFromJavaClass(Definition.getBoxedType(prefix.actual)); + struct = locals.getPainlessLookup().getPainlessStructFromJavaClass(PainlessLookup.getBoxedType(prefix.actual)); } - MethodKey methodKey = new MethodKey(name, arguments.size()); - Method method = prefix instanceof EStatic ? struct.staticMethods.get(methodKey) : struct.methods.get(methodKey); + PainlessMethodKey methodKey = new PainlessMethodKey(name, arguments.size()); + PainlessMethod method = prefix instanceof EStatic ? struct.staticMethods.get(methodKey) : struct.methods.get(methodKey); if (method != null) { sub = new PSubCallInvoke(location, method, prefix.actual, arguments); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java index de2c05dfa9b28..8d27162fc367b 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PField.java @@ -19,15 +19,16 @@ package org.elasticsearch.painless.node; -import org.elasticsearch.painless.Definition; -import org.elasticsearch.painless.Definition.Field; -import org.elasticsearch.painless.Definition.Method; -import org.elasticsearch.painless.Definition.Struct; -import org.elasticsearch.painless.Definition.def; +import org.elasticsearch.painless.lookup.PainlessLookup; +import org.elasticsearch.painless.lookup.PainlessField; +import org.elasticsearch.painless.lookup.PainlessMethod; +import org.elasticsearch.painless.lookup.PainlessClass; +import org.elasticsearch.painless.lookup.PainlessLookup.def; import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.MethodWriter; +import org.elasticsearch.painless.lookup.PainlessMethodKey; import java.util.List; import java.util.Map; @@ -63,29 +64,29 @@ void analyze(Locals locals) { prefix = prefix.cast(locals); if (prefix.actual.isArray()) { - sub = new PSubArrayLength(location, Definition.ClassToName(prefix.actual), value); + sub = new PSubArrayLength(location, PainlessLookup.ClassToName(prefix.actual), value); } else if (prefix.actual == def.class) { sub = new PSubDefField(location, value); } else { - Struct struct = locals.getDefinition().getPainlessStructFromJavaClass(prefix.actual); - Field field = prefix instanceof EStatic ? struct.staticMembers.get(value) : struct.members.get(value); + PainlessClass struct = locals.getPainlessLookup().getPainlessStructFromJavaClass(prefix.actual); + PainlessField field = prefix instanceof EStatic ? struct.staticMembers.get(value) : struct.members.get(value); if (field != null) { sub = new PSubField(location, field); } else { - Method getter = struct.methods.get( - new Definition.MethodKey("get" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 0)); + PainlessMethod getter = struct.methods.get( + new PainlessMethodKey("get" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 0)); if (getter == null) { getter = struct.methods.get( - new Definition.MethodKey("is" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 0)); + new PainlessMethodKey("is" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 0)); } - Method setter = struct.methods.get( - new Definition.MethodKey("set" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 1)); + PainlessMethod setter = struct.methods.get( + new PainlessMethodKey("set" + Character.toUpperCase(value.charAt(0)) + value.substring(1), 1)); if (getter != null || setter != null) { - sub = new PSubShortcut(location, value, Definition.ClassToName(prefix.actual), getter, setter); + sub = new PSubShortcut(location, value, PainlessLookup.ClassToName(prefix.actual), getter, setter); } else { EConstant index = new EConstant(location, value); index.analyze(locals); @@ -103,7 +104,7 @@ void analyze(Locals locals) { if (sub == null) { throw createError(new IllegalArgumentException( - "Unknown field [" + value + "] for type [" + Definition.ClassToName(prefix.actual) + "].")); + "Unknown field [" + value + "] for type [" + PainlessLookup.ClassToName(prefix.actual) + "].")); } if (nullSafe) { diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java index e13fe0d85c143..66ad0ecff1b89 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubCallInvoke.java @@ -19,7 +19,7 @@ package org.elasticsearch.painless.node; -import org.elasticsearch.painless.Definition.Method; +import org.elasticsearch.painless.lookup.PainlessMethod; import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Location; @@ -34,11 +34,11 @@ */ final class PSubCallInvoke extends AExpression { - private final Method method; + private final PainlessMethod method; private final Class box; private final List arguments; - PSubCallInvoke(Location location, Method method, Class box, List arguments) { + PSubCallInvoke(Location location, PainlessMethod method, Class box, List arguments) { super(location); this.method = Objects.requireNonNull(method); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefArray.java index 27087928d4fec..8e30d43432953 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefArray.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefArray.java @@ -20,7 +20,7 @@ package org.elasticsearch.painless.node; import org.elasticsearch.painless.DefBootstrap; -import org.elasticsearch.painless.Definition.def; +import org.elasticsearch.painless.lookup.PainlessLookup.def; import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Location; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java index 6428e47d1bacc..0882f19177006 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefCall.java @@ -20,7 +20,7 @@ package org.elasticsearch.painless.node; import org.elasticsearch.painless.DefBootstrap; -import org.elasticsearch.painless.Definition.def; +import org.elasticsearch.painless.lookup.PainlessLookup.def; import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Location; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java index fbdfc47a65e66..41fcf563d241c 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubDefField.java @@ -20,7 +20,7 @@ package org.elasticsearch.painless.node; import org.elasticsearch.painless.DefBootstrap; -import org.elasticsearch.painless.Definition.def; +import org.elasticsearch.painless.lookup.PainlessLookup.def; import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Location; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java index b386feacb846d..d6c367cfeabec 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubField.java @@ -19,8 +19,8 @@ package org.elasticsearch.painless.node; -import org.elasticsearch.painless.Definition; -import org.elasticsearch.painless.Definition.Field; +import org.elasticsearch.painless.lookup.PainlessLookup; +import org.elasticsearch.painless.lookup.PainlessField; import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Location; @@ -35,9 +35,9 @@ */ final class PSubField extends AStoreable { - private final Field field; + private final PainlessField field; - PSubField(Location location, Field field) { + PSubField(Location location, PainlessField field) { super(location); this.field = Objects.requireNonNull(field); @@ -52,7 +52,7 @@ void extractVariables(Set variables) { void analyze(Locals locals) { if (write && Modifier.isFinal(field.modifiers)) { throw createError(new IllegalArgumentException( - "Cannot write to read-only field [" + field.name + "] for type [" + Definition.ClassToName(field.clazz) + "].")); + "Cannot write to read-only field [" + field.name + "] for type [" + PainlessLookup.ClassToName(field.clazz) + "].")); } actual = field.clazz; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java index 194a43e16dd16..5d881b30db22d 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubListShortcut.java @@ -19,14 +19,14 @@ package org.elasticsearch.painless.node; -import org.elasticsearch.painless.Definition; -import org.elasticsearch.painless.Definition.Method; -import org.elasticsearch.painless.Definition.Struct; +import org.elasticsearch.painless.lookup.PainlessMethod; +import org.elasticsearch.painless.lookup.PainlessClass; import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.MethodWriter; import org.elasticsearch.painless.WriterConstants; +import org.elasticsearch.painless.lookup.PainlessMethodKey; import java.util.Objects; import java.util.Set; @@ -36,13 +36,13 @@ */ final class PSubListShortcut extends AStoreable { - private final Struct struct; + private final PainlessClass struct; private AExpression index; - private Method getter; - private Method setter; + private PainlessMethod getter; + private PainlessMethod setter; - PSubListShortcut(Location location, Struct struct, AExpression index) { + PSubListShortcut(Location location, PainlessClass struct, AExpression index) { super(location); this.struct = Objects.requireNonNull(struct); @@ -56,8 +56,8 @@ void extractVariables(Set variables) { @Override void analyze(Locals locals) { - getter = struct.methods.get(new Definition.MethodKey("get", 1)); - setter = struct.methods.get(new Definition.MethodKey("set", 2)); + getter = struct.methods.get(new PainlessMethodKey("get", 1)); + setter = struct.methods.get(new PainlessMethodKey("set", 2)); if (getter != null && (getter.rtn == void.class || getter.arguments.size() != 1 || getter.arguments.get(0) != int.class)) { diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java index 033db404640e4..4875d55cbeb58 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubMapShortcut.java @@ -19,13 +19,13 @@ package org.elasticsearch.painless.node; -import org.elasticsearch.painless.Definition; -import org.elasticsearch.painless.Definition.Method; -import org.elasticsearch.painless.Definition.Struct; +import org.elasticsearch.painless.lookup.PainlessMethod; +import org.elasticsearch.painless.lookup.PainlessClass; import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Location; import org.elasticsearch.painless.MethodWriter; +import org.elasticsearch.painless.lookup.PainlessMethodKey; import java.util.Objects; import java.util.Set; @@ -35,13 +35,13 @@ */ final class PSubMapShortcut extends AStoreable { - private final Struct struct; + private final PainlessClass struct; private AExpression index; - private Method getter; - private Method setter; + private PainlessMethod getter; + private PainlessMethod setter; - PSubMapShortcut(Location location, Struct struct, AExpression index) { + PSubMapShortcut(Location location, PainlessClass struct, AExpression index) { super(location); this.struct = Objects.requireNonNull(struct); @@ -55,8 +55,8 @@ void extractVariables(Set variables) { @Override void analyze(Locals locals) { - getter = struct.methods.get(new Definition.MethodKey("get", 1)); - setter = struct.methods.get(new Definition.MethodKey("put", 2)); + getter = struct.methods.get(new PainlessMethodKey("get", 1)); + setter = struct.methods.get(new PainlessMethodKey("put", 2)); if (getter != null && (getter.rtn == void.class || getter.arguments.size() != 1)) { throw createError(new IllegalArgumentException("Illegal map get shortcut for type [" + struct.name + "].")); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubShortcut.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubShortcut.java index ff88f0018556c..4b2910dbc010e 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubShortcut.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/PSubShortcut.java @@ -19,7 +19,7 @@ package org.elasticsearch.painless.node; -import org.elasticsearch.painless.Definition.Method; +import org.elasticsearch.painless.lookup.PainlessMethod; import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Location; @@ -34,10 +34,10 @@ final class PSubShortcut extends AStoreable { private final String value; private final String type; - private final Method getter; - private final Method setter; + private final PainlessMethod getter; + private final PainlessMethod setter; - PSubShortcut(Location location, String value, String type, Method getter, Method setter) { + PSubShortcut(Location location, String value, String type, PainlessMethod getter, PainlessMethod setter) { super(location); this.value = value; diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java index 98e45ca29f416..8a703c80cba2f 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SCatch.java @@ -67,7 +67,7 @@ void analyze(Locals locals) { Class clazz; try { - clazz = locals.getDefinition().getJavaClassFromPainlessType(this.type); + clazz = locals.getPainlessLookup().getJavaClassFromPainlessType(this.type); } catch (IllegalArgumentException exception) { throw createError(new IllegalArgumentException("Not a type [" + this.type + "].")); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java index 9f3f86abf438b..fb92c20e89e01 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SDeclaration.java @@ -62,7 +62,7 @@ void analyze(Locals locals) { Class clazz; try { - clazz = locals.getDefinition().getJavaClassFromPainlessType(this.type); + clazz = locals.getPainlessLookup().getJavaClassFromPainlessType(this.type); } catch (IllegalArgumentException exception) { throw createError(new IllegalArgumentException("Not a type [" + this.type + "].")); } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java index a3c8319825a26..e7d18ece0590d 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SEach.java @@ -19,8 +19,8 @@ package org.elasticsearch.painless.node; -import org.elasticsearch.painless.Definition; -import org.elasticsearch.painless.Definition.def; +import org.elasticsearch.painless.lookup.PainlessLookup; +import org.elasticsearch.painless.lookup.PainlessLookup.def; import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Locals.Variable; @@ -71,7 +71,7 @@ void analyze(Locals locals) { Class clazz; try { - clazz = locals.getDefinition().getJavaClassFromPainlessType(this.type); + clazz = locals.getPainlessLookup().getJavaClassFromPainlessType(this.type); } catch (IllegalArgumentException exception) { throw createError(new IllegalArgumentException("Not a type [" + this.type + "].")); } @@ -84,7 +84,8 @@ void analyze(Locals locals) { } else if (expression.actual == def.class || Iterable.class.isAssignableFrom(expression.actual)) { sub = new SSubEachIterable(location, variable, expression, block); } else { - throw createError(new IllegalArgumentException("Illegal for each type [" + Definition.ClassToName(expression.actual) + "].")); + throw createError( + new IllegalArgumentException("Illegal for each type [" + PainlessLookup.ClassToName(expression.actual) + "].")); } sub.analyze(locals); diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java index 1b1e6bd2ef84b..628bb1d32d59e 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SFunction.java @@ -22,8 +22,8 @@ import org.elasticsearch.painless.CompilerSettings; import org.elasticsearch.painless.Constant; import org.elasticsearch.painless.Def; -import org.elasticsearch.painless.Definition; -import org.elasticsearch.painless.Definition.Method; +import org.elasticsearch.painless.lookup.PainlessLookup; +import org.elasticsearch.painless.lookup.PainlessMethod; import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Locals.Parameter; @@ -93,7 +93,7 @@ public int getMaxLoopCounter() { Class rtnType = null; List parameters = new ArrayList<>(); - Method method = null; + PainlessMethod method = null; private Variable loop = null; @@ -117,9 +117,9 @@ void extractVariables(Set variables) { throw new IllegalStateException("Illegal tree structure"); } - void generateSignature(Definition definition) { + void generateSignature(PainlessLookup painlessLookup) { try { - rtnType = definition.getJavaClassFromPainlessType(rtnTypeStr); + rtnType = painlessLookup.getJavaClassFromPainlessType(rtnTypeStr); } catch (IllegalArgumentException exception) { throw createError(new IllegalArgumentException("Illegal return type [" + rtnTypeStr + "] for function [" + name + "].")); } @@ -133,9 +133,9 @@ void generateSignature(Definition definition) { for (int param = 0; param < this.paramTypeStrs.size(); ++param) { try { - Class paramType = definition.getJavaClassFromPainlessType(this.paramTypeStrs.get(param)); + Class paramType = painlessLookup.getJavaClassFromPainlessType(this.paramTypeStrs.get(param)); - paramClasses[param] = Definition.defClassToObjectClass(paramType); + paramClasses[param] = PainlessLookup.defClassToObjectClass(paramType); paramTypes.add(paramType); parameters.add(new Parameter(location, paramNameStrs.get(param), paramType)); } catch (IllegalArgumentException exception) { @@ -145,8 +145,8 @@ void generateSignature(Definition definition) { } org.objectweb.asm.commons.Method method = new org.objectweb.asm.commons.Method( - name, MethodType.methodType(Definition.defClassToObjectClass(rtnType), paramClasses).toMethodDescriptorString()); - this.method = new Method(name, null, null, rtnType, paramTypes, method, Modifier.STATIC | Modifier.PRIVATE, null); + name, MethodType.methodType(PainlessLookup.defClassToObjectClass(rtnType), paramClasses).toMethodDescriptorString()); + this.method = new PainlessMethod(name, null, null, rtnType, paramTypes, method, Modifier.STATIC | Modifier.PRIVATE, null); } @Override diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java index efb6db278140d..4781457a57dfa 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSource.java @@ -21,9 +21,9 @@ import org.elasticsearch.painless.CompilerSettings; import org.elasticsearch.painless.Constant; -import org.elasticsearch.painless.Definition; -import org.elasticsearch.painless.Definition.Method; -import org.elasticsearch.painless.Definition.MethodKey; +import org.elasticsearch.painless.lookup.PainlessLookup; +import org.elasticsearch.painless.lookup.PainlessMethod; +import org.elasticsearch.painless.lookup.PainlessMethodKey; import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Locals.Variable; @@ -167,20 +167,20 @@ void extractVariables(Set variables) { throw new IllegalStateException("Illegal tree structure."); } - public void analyze(Definition definition) { - Map methods = new HashMap<>(); + public void analyze(PainlessLookup painlessLookup) { + Map methods = new HashMap<>(); for (SFunction function : functions) { - function.generateSignature(definition); + function.generateSignature(painlessLookup); - MethodKey key = new MethodKey(function.name, function.parameters.size()); + PainlessMethodKey key = new PainlessMethodKey(function.name, function.parameters.size()); if (methods.put(key, function.method) != null) { throw createError(new IllegalArgumentException("Duplicate functions with name [" + function.name + "].")); } } - analyze(Locals.newProgramScope(definition, methods.values())); + analyze(Locals.newProgramScope(painlessLookup, methods.values())); } @Override diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java index e0c9476ba640a..5db161b8002a4 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachArray.java @@ -20,8 +20,8 @@ package org.elasticsearch.painless.node; import org.elasticsearch.painless.AnalyzerCaster; -import org.elasticsearch.painless.Definition; -import org.elasticsearch.painless.Definition.Cast; +import org.elasticsearch.painless.lookup.PainlessLookup; +import org.elasticsearch.painless.lookup.PainlessCast; import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Locals.Variable; @@ -41,7 +41,7 @@ final class SSubEachArray extends AStatement { private AExpression expression; private final SBlock block; - private Cast cast = null; + private PainlessCast cast = null; private Variable array = null; private Variable index = null; private Class indexed = null; @@ -109,6 +109,6 @@ void write(MethodWriter writer, Globals globals) { @Override public String toString() { - return singleLineToString(Definition.ClassToName(variable.clazz), variable.name, expression, block); + return singleLineToString(PainlessLookup.ClassToName(variable.clazz), variable.name, expression, block); } } diff --git a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java index 11e0f15d7e4f8..faee2ed74a6d0 100644 --- a/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java +++ b/modules/lang-painless/src/main/java/org/elasticsearch/painless/node/SSubEachIterable.java @@ -21,11 +21,11 @@ import org.elasticsearch.painless.AnalyzerCaster; import org.elasticsearch.painless.DefBootstrap; -import org.elasticsearch.painless.Definition; -import org.elasticsearch.painless.Definition.Cast; -import org.elasticsearch.painless.Definition.Method; -import org.elasticsearch.painless.Definition.MethodKey; -import org.elasticsearch.painless.Definition.def; +import org.elasticsearch.painless.lookup.PainlessLookup; +import org.elasticsearch.painless.lookup.PainlessCast; +import org.elasticsearch.painless.lookup.PainlessMethod; +import org.elasticsearch.painless.lookup.PainlessMethodKey; +import org.elasticsearch.painless.lookup.PainlessLookup.def; import org.elasticsearch.painless.Globals; import org.elasticsearch.painless.Locals; import org.elasticsearch.painless.Locals.Variable; @@ -51,9 +51,9 @@ final class SSubEachIterable extends AStatement { private final SBlock block; private final Variable variable; - private Cast cast = null; + private PainlessCast cast = null; private Variable iterator = null; - private Method method = null; + private PainlessMethod method = null; SSubEachIterable(Location location, Variable variable, AExpression expression, SBlock block) { super(location); @@ -77,11 +77,12 @@ void analyze(Locals locals) { if (expression.actual == def.class) { method = null; } else { - method = locals.getDefinition().getPainlessStructFromJavaClass(expression.actual).methods.get(new MethodKey("iterator", 0)); + method = locals.getPainlessLookup(). + getPainlessStructFromJavaClass(expression.actual).methods.get(new PainlessMethodKey("iterator", 0)); if (method == null) { throw createError(new IllegalArgumentException( - "Unable to create iterator for the type [" + Definition.ClassToName(expression.actual) + "].")); + "Unable to create iterator for the type [" + PainlessLookup.ClassToName(expression.actual) + "].")); } } @@ -132,6 +133,6 @@ void write(MethodWriter writer, Globals globals) { @Override public String toString() { - return singleLineToString(Definition.ClassToName(variable.clazz), variable.name, expression, block); + return singleLineToString(PainlessLookup.ClassToName(variable.clazz), variable.name, expression, block); } } diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AnalyzerCasterTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AnalyzerCasterTests.java index b369d8beca764..a702490fff9d4 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/AnalyzerCasterTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/AnalyzerCasterTests.java @@ -19,7 +19,7 @@ package org.elasticsearch.painless; -import org.elasticsearch.painless.Definition.Cast; +import org.elasticsearch.painless.lookup.PainlessCast; import org.elasticsearch.test.ESTestCase; @@ -35,7 +35,7 @@ private static void assertCast(Class actual, Class expected, boolean mustB return; } - Cast cast = AnalyzerCaster.getLegalCast(location, actual, expected, true, false); + PainlessCast cast = AnalyzerCaster.getLegalCast(location, actual, expected, true, false); assertEquals(actual, cast.from); assertEquals(expected, cast.to); diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BaseClassTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BaseClassTests.java index 59cafa96ddcb9..78e5814e963f7 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/BaseClassTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/BaseClassTests.java @@ -23,6 +23,7 @@ import java.util.HashMap; import java.util.Map; +import org.elasticsearch.painless.lookup.PainlessLookup; import org.elasticsearch.painless.spi.Whitelist; import static java.util.Collections.emptyMap; @@ -36,7 +37,7 @@ */ public class BaseClassTests extends ScriptTestCase { - private final Definition definition = new Definition(Whitelist.BASE_WHITELISTS); + private final PainlessLookup painlessLookup = new PainlessLookup(Whitelist.BASE_WHITELISTS); public abstract static class Gets { @@ -67,7 +68,7 @@ public Map getTestMap() { } public void testGets() { - Compiler compiler = new Compiler(Gets.class, definition); + Compiler compiler = new Compiler(Gets.class, painlessLookup); Map map = new HashMap<>(); map.put("s", 1); @@ -85,7 +86,7 @@ public abstract static class NoArgs { public abstract Object execute(); } public void testNoArgs() { - Compiler compiler = new Compiler(NoArgs.class, definition); + Compiler compiler = new Compiler(NoArgs.class, painlessLookup); assertEquals(1, ((NoArgs)scriptEngine.compile(compiler, null, "1", emptyMap())).execute()); assertEquals("foo", ((NoArgs)scriptEngine.compile(compiler, null, "'foo'", emptyMap())).execute()); @@ -109,13 +110,13 @@ public abstract static class OneArg { public abstract Object execute(Object arg); } public void testOneArg() { - Compiler compiler = new Compiler(OneArg.class, definition); + Compiler compiler = new Compiler(OneArg.class, painlessLookup); Object rando = randomInt(); assertEquals(rando, ((OneArg)scriptEngine.compile(compiler, null, "arg", emptyMap())).execute(rando)); rando = randomAlphaOfLength(5); assertEquals(rando, ((OneArg)scriptEngine.compile(compiler, null, "arg", emptyMap())).execute(rando)); - Compiler noargs = new Compiler(NoArgs.class, definition); + Compiler noargs = new Compiler(NoArgs.class, painlessLookup); Exception e = expectScriptThrows(IllegalArgumentException.class, () -> scriptEngine.compile(noargs, null, "doc", emptyMap())); assertEquals("Variable [doc] is not defined.", e.getMessage()); @@ -130,7 +131,7 @@ public abstract static class ArrayArg { public abstract Object execute(String[] arg); } public void testArrayArg() { - Compiler compiler = new Compiler(ArrayArg.class, definition); + Compiler compiler = new Compiler(ArrayArg.class, painlessLookup); String rando = randomAlphaOfLength(5); assertEquals(rando, ((ArrayArg)scriptEngine.compile(compiler, null, "arg[0]", emptyMap())).execute(new String[] {rando, "foo"})); } @@ -140,7 +141,7 @@ public abstract static class PrimitiveArrayArg { public abstract Object execute(int[] arg); } public void testPrimitiveArrayArg() { - Compiler compiler = new Compiler(PrimitiveArrayArg.class, definition); + Compiler compiler = new Compiler(PrimitiveArrayArg.class, painlessLookup); int rando = randomInt(); assertEquals(rando, ((PrimitiveArrayArg)scriptEngine.compile(compiler, null, "arg[0]", emptyMap())).execute(new int[] {rando, 10})); } @@ -150,7 +151,7 @@ public abstract static class DefArrayArg { public abstract Object execute(Object[] arg); } public void testDefArrayArg() { - Compiler compiler = new Compiler(DefArrayArg.class, definition); + Compiler compiler = new Compiler(DefArrayArg.class, painlessLookup); Object rando = randomInt(); assertEquals(rando, ((DefArrayArg)scriptEngine.compile(compiler, null, "arg[0]", emptyMap())).execute(new Object[] {rando, 10})); rando = randomAlphaOfLength(5); @@ -168,7 +169,7 @@ public abstract static class ManyArgs { public abstract boolean needsD(); } public void testManyArgs() { - Compiler compiler = new Compiler(ManyArgs.class, definition); + Compiler compiler = new Compiler(ManyArgs.class, painlessLookup); int rando = randomInt(); assertEquals(rando, ((ManyArgs)scriptEngine.compile(compiler, null, "a", emptyMap())).execute(rando, 0, 0, 0)); assertEquals(10, ((ManyArgs)scriptEngine.compile(compiler, null, "a + b + c + d", emptyMap())).execute(1, 2, 3, 4)); @@ -196,7 +197,7 @@ public abstract static class VarargTest { public abstract Object execute(String... arg); } public void testVararg() { - Compiler compiler = new Compiler(VarargTest.class, definition); + Compiler compiler = new Compiler(VarargTest.class, painlessLookup); assertEquals("foo bar baz", ((VarargTest)scriptEngine.compile(compiler, null, "String.join(' ', Arrays.asList(arg))", emptyMap())) .execute("foo", "bar", "baz")); } @@ -212,7 +213,7 @@ public Object executeWithASingleOne(int a, int b, int c) { } } public void testDefaultMethods() { - Compiler compiler = new Compiler(DefaultMethods.class, definition); + Compiler compiler = new Compiler(DefaultMethods.class, painlessLookup); int rando = randomInt(); assertEquals(rando, ((DefaultMethods)scriptEngine.compile(compiler, null, "a", emptyMap())).execute(rando, 0, 0, 0)); assertEquals(rando, ((DefaultMethods)scriptEngine.compile(compiler, null, "a", emptyMap())).executeWithASingleOne(rando, 0, 0)); @@ -226,7 +227,7 @@ public abstract static class ReturnsVoid { public abstract void execute(Map map); } public void testReturnsVoid() { - Compiler compiler = new Compiler(ReturnsVoid.class, definition); + Compiler compiler = new Compiler(ReturnsVoid.class, painlessLookup); Map map = new HashMap<>(); ((ReturnsVoid)scriptEngine.compile(compiler, null, "map.a = 'foo'", emptyMap())).execute(map); assertEquals(singletonMap("a", "foo"), map); @@ -245,7 +246,7 @@ public abstract static class ReturnsPrimitiveBoolean { public abstract boolean execute(); } public void testReturnsPrimitiveBoolean() { - Compiler compiler = new Compiler(ReturnsPrimitiveBoolean.class, definition); + Compiler compiler = new Compiler(ReturnsPrimitiveBoolean.class, painlessLookup); assertEquals(true, ((ReturnsPrimitiveBoolean)scriptEngine.compile(compiler, null, "true", emptyMap())).execute()); assertEquals(false, ((ReturnsPrimitiveBoolean)scriptEngine.compile(compiler, null, "false", emptyMap())).execute()); @@ -287,7 +288,7 @@ public abstract static class ReturnsPrimitiveInt { public abstract int execute(); } public void testReturnsPrimitiveInt() { - Compiler compiler = new Compiler(ReturnsPrimitiveInt.class, definition); + Compiler compiler = new Compiler(ReturnsPrimitiveInt.class, painlessLookup); assertEquals(1, ((ReturnsPrimitiveInt)scriptEngine.compile(compiler, null, "1", emptyMap())).execute()); assertEquals(1, ((ReturnsPrimitiveInt)scriptEngine.compile(compiler, null, "(int) 1L", emptyMap())).execute()); @@ -329,7 +330,7 @@ public abstract static class ReturnsPrimitiveFloat { public abstract float execute(); } public void testReturnsPrimitiveFloat() { - Compiler compiler = new Compiler(ReturnsPrimitiveFloat.class, definition); + Compiler compiler = new Compiler(ReturnsPrimitiveFloat.class, painlessLookup); assertEquals(1.1f, ((ReturnsPrimitiveFloat)scriptEngine.compile(compiler, null, "1.1f", emptyMap())).execute(), 0); assertEquals(1.1f, ((ReturnsPrimitiveFloat)scriptEngine.compile(compiler, null, "(float) 1.1d", emptyMap())).execute(), 0); @@ -360,7 +361,7 @@ public abstract static class ReturnsPrimitiveDouble { public abstract double execute(); } public void testReturnsPrimitiveDouble() { - Compiler compiler = new Compiler(ReturnsPrimitiveDouble.class, definition); + Compiler compiler = new Compiler(ReturnsPrimitiveDouble.class, painlessLookup); assertEquals(1.0, ((ReturnsPrimitiveDouble)scriptEngine.compile(compiler, null, "1", emptyMap())).execute(), 0); assertEquals(1.0, ((ReturnsPrimitiveDouble)scriptEngine.compile(compiler, null, "1L", emptyMap())).execute(), 0); @@ -394,7 +395,7 @@ public abstract static class NoArgumentsConstant { public abstract Object execute(String foo); } public void testNoArgumentsConstant() { - Compiler compiler = new Compiler(NoArgumentsConstant.class, definition); + Compiler compiler = new Compiler(NoArgumentsConstant.class, painlessLookup); Exception e = expectScriptThrows(IllegalArgumentException.class, false, () -> scriptEngine.compile(compiler, null, "1", emptyMap())); assertThat(e.getMessage(), startsWith( @@ -407,7 +408,7 @@ public abstract static class WrongArgumentsConstant { public abstract Object execute(String foo); } public void testWrongArgumentsConstant() { - Compiler compiler = new Compiler(WrongArgumentsConstant.class, definition); + Compiler compiler = new Compiler(WrongArgumentsConstant.class, painlessLookup); Exception e = expectScriptThrows(IllegalArgumentException.class, false, () -> scriptEngine.compile(compiler, null, "1", emptyMap())); assertThat(e.getMessage(), startsWith( @@ -420,7 +421,7 @@ public abstract static class WrongLengthOfArgumentConstant { public abstract Object execute(String foo); } public void testWrongLengthOfArgumentConstant() { - Compiler compiler = new Compiler(WrongLengthOfArgumentConstant.class, definition); + Compiler compiler = new Compiler(WrongLengthOfArgumentConstant.class, painlessLookup); Exception e = expectScriptThrows(IllegalArgumentException.class, false, () -> scriptEngine.compile(compiler, null, "1", emptyMap())); assertThat(e.getMessage(), startsWith("[" + WrongLengthOfArgumentConstant.class.getName() + "#ARGUMENTS] has length [2] but [" @@ -432,7 +433,7 @@ public abstract static class UnknownArgType { public abstract Object execute(UnknownArgType foo); } public void testUnknownArgType() { - Compiler compiler = new Compiler(UnknownArgType.class, definition); + Compiler compiler = new Compiler(UnknownArgType.class, painlessLookup); Exception e = expectScriptThrows(IllegalArgumentException.class, false, () -> scriptEngine.compile(compiler, null, "1", emptyMap())); assertEquals("[foo] is of unknown type [" + UnknownArgType.class.getName() + ". Painless interfaces can only accept arguments " @@ -444,7 +445,7 @@ public abstract static class UnknownReturnType { public abstract UnknownReturnType execute(String foo); } public void testUnknownReturnType() { - Compiler compiler = new Compiler(UnknownReturnType.class, definition); + Compiler compiler = new Compiler(UnknownReturnType.class, painlessLookup); Exception e = expectScriptThrows(IllegalArgumentException.class, false, () -> scriptEngine.compile(compiler, null, "1", emptyMap())); assertEquals("Painless can only implement execute methods returning a whitelisted type but [" + UnknownReturnType.class.getName() @@ -456,7 +457,7 @@ public abstract static class UnknownArgTypeInArray { public abstract Object execute(UnknownArgTypeInArray[] foo); } public void testUnknownArgTypeInArray() { - Compiler compiler = new Compiler(UnknownArgTypeInArray.class, definition); + Compiler compiler = new Compiler(UnknownArgTypeInArray.class, painlessLookup); Exception e = expectScriptThrows(IllegalArgumentException.class, false, () -> scriptEngine.compile(compiler, null, "1", emptyMap())); assertEquals("[foo] is of unknown type [" + UnknownArgTypeInArray.class.getName() + ". Painless interfaces can only accept " @@ -468,7 +469,7 @@ public abstract static class TwoExecuteMethods { public abstract Object execute(boolean foo); } public void testTwoExecuteMethods() { - Compiler compiler = new Compiler(TwoExecuteMethods.class, definition); + Compiler compiler = new Compiler(TwoExecuteMethods.class, painlessLookup); Exception e = expectScriptThrows(IllegalArgumentException.class, false, () -> scriptEngine.compile(compiler, null, "null", emptyMap())); assertEquals("Painless can only implement interfaces that have a single method named [execute] but [" diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DebugTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DebugTests.java index 279438e74a7c3..987eef31eeeaf 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DebugTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DebugTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.ElasticsearchException; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.painless.lookup.PainlessLookup; import org.elasticsearch.painless.spi.Whitelist; import org.elasticsearch.script.ScriptException; @@ -35,7 +36,7 @@ import static org.hamcrest.Matchers.not; public class DebugTests extends ScriptTestCase { - private final Definition definition = new Definition(Whitelist.BASE_WHITELISTS); + private final PainlessLookup painlessLookup = new PainlessLookup(Whitelist.BASE_WHITELISTS); public void testExplain() { // Debug.explain can explain an object @@ -43,16 +44,16 @@ public void testExplain() { PainlessExplainError e = expectScriptThrows(PainlessExplainError.class, () -> exec( "Debug.explain(params.a)", singletonMap("a", dummy), true)); assertSame(dummy, e.getObjectToExplain()); - assertThat(e.getHeaders(definition), hasEntry("es.to_string", singletonList(dummy.toString()))); - assertThat(e.getHeaders(definition), hasEntry("es.java_class", singletonList("java.lang.Object"))); - assertThat(e.getHeaders(definition), hasEntry("es.painless_class", singletonList("java.lang.Object"))); + assertThat(e.getHeaders(painlessLookup), hasEntry("es.to_string", singletonList(dummy.toString()))); + assertThat(e.getHeaders(painlessLookup), hasEntry("es.java_class", singletonList("java.lang.Object"))); + assertThat(e.getHeaders(painlessLookup), hasEntry("es.painless_class", singletonList("java.lang.Object"))); // Null should be ok e = expectScriptThrows(PainlessExplainError.class, () -> exec("Debug.explain(null)")); assertNull(e.getObjectToExplain()); - assertThat(e.getHeaders(definition), hasEntry("es.to_string", singletonList("null"))); - assertThat(e.getHeaders(definition), not(hasKey("es.java_class"))); - assertThat(e.getHeaders(definition), not(hasKey("es.painless_class"))); + assertThat(e.getHeaders(painlessLookup), hasEntry("es.to_string", singletonList("null"))); + assertThat(e.getHeaders(painlessLookup), not(hasKey("es.java_class"))); + assertThat(e.getHeaders(painlessLookup), not(hasKey("es.painless_class"))); // You can't catch the explain exception e = expectScriptThrows(PainlessExplainError.class, () -> exec( diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java index d17b9e55ab0c1..0d5e2748b7b32 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/Debugger.java @@ -19,6 +19,7 @@ package org.elasticsearch.painless; +import org.elasticsearch.painless.lookup.PainlessLookup; import org.elasticsearch.painless.spi.Whitelist; import org.objectweb.asm.util.Textifier; @@ -39,7 +40,7 @@ static String toString(Class iface, String source, CompilerSettings settings) PrintWriter outputWriter = new PrintWriter(output); Textifier textifier = new Textifier(); try { - new Compiler(iface, new Definition(Whitelist.BASE_WHITELISTS)) + new Compiler(iface, new PainlessLookup(Whitelist.BASE_WHITELISTS)) .compile("", source, settings, textifier); } catch (RuntimeException e) { textifier.print(outputWriter); diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java index 291a4807606ea..077f151fe721d 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/DefBootstrapTests.java @@ -27,15 +27,16 @@ import java.util.Collections; import java.util.HashMap; +import org.elasticsearch.painless.lookup.PainlessLookup; import org.elasticsearch.painless.spi.Whitelist; import org.elasticsearch.test.ESTestCase; public class DefBootstrapTests extends ESTestCase { - private final Definition definition = new Definition(Whitelist.BASE_WHITELISTS); + private final PainlessLookup painlessLookup = new PainlessLookup(Whitelist.BASE_WHITELISTS); /** calls toString() on integers, twice */ public void testOneType() throws Throwable { - CallSite site = DefBootstrap.bootstrap(definition, + CallSite site = DefBootstrap.bootstrap(painlessLookup, MethodHandles.publicLookup(), "toString", MethodType.methodType(String.class, Object.class), @@ -55,7 +56,7 @@ public void testOneType() throws Throwable { } public void testTwoTypes() throws Throwable { - CallSite site = DefBootstrap.bootstrap(definition, + CallSite site = DefBootstrap.bootstrap(painlessLookup, MethodHandles.publicLookup(), "toString", MethodType.methodType(String.class, Object.class), @@ -80,7 +81,7 @@ public void testTwoTypes() throws Throwable { public void testTooManyTypes() throws Throwable { // if this changes, test must be rewritten assertEquals(5, DefBootstrap.PIC.MAX_DEPTH); - CallSite site = DefBootstrap.bootstrap(definition, + CallSite site = DefBootstrap.bootstrap(painlessLookup, MethodHandles.publicLookup(), "toString", MethodType.methodType(String.class, Object.class), @@ -106,7 +107,7 @@ public void testTooManyTypes() throws Throwable { /** test that we revert to the megamorphic classvalue cache and that it works as expected */ public void testMegamorphic() throws Throwable { - DefBootstrap.PIC site = (DefBootstrap.PIC) DefBootstrap.bootstrap(definition, + DefBootstrap.PIC site = (DefBootstrap.PIC) DefBootstrap.bootstrap(painlessLookup, MethodHandles.publicLookup(), "size", MethodType.methodType(int.class, Object.class), @@ -138,7 +139,7 @@ public void testMegamorphic() throws Throwable { // test operators with null guards public void testNullGuardAdd() throws Throwable { - DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(definition, + DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(painlessLookup, MethodHandles.publicLookup(), "add", MethodType.methodType(Object.class, Object.class, Object.class), @@ -150,7 +151,7 @@ public void testNullGuardAdd() throws Throwable { } public void testNullGuardAddWhenCached() throws Throwable { - DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(definition, + DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(painlessLookup, MethodHandles.publicLookup(), "add", MethodType.methodType(Object.class, Object.class, Object.class), @@ -163,7 +164,7 @@ public void testNullGuardAddWhenCached() throws Throwable { } public void testNullGuardEq() throws Throwable { - DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(definition, + DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(painlessLookup, MethodHandles.publicLookup(), "eq", MethodType.methodType(boolean.class, Object.class, Object.class), @@ -176,7 +177,7 @@ public void testNullGuardEq() throws Throwable { } public void testNullGuardEqWhenCached() throws Throwable { - DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(definition, + DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(painlessLookup, MethodHandles.publicLookup(), "eq", MethodType.methodType(boolean.class, Object.class, Object.class), @@ -194,7 +195,7 @@ public void testNullGuardEqWhenCached() throws Throwable { // and can be disabled in some circumstances. public void testNoNullGuardAdd() throws Throwable { - DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(definition, + DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(painlessLookup, MethodHandles.publicLookup(), "add", MethodType.methodType(Object.class, int.class, Object.class), @@ -208,7 +209,7 @@ public void testNoNullGuardAdd() throws Throwable { } public void testNoNullGuardAddWhenCached() throws Throwable { - DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(definition, + DefBootstrap.MIC site = (DefBootstrap.MIC) DefBootstrap.bootstrap(painlessLookup, MethodHandles.publicLookup(), "add", MethodType.methodType(Object.class, int.class, Object.class), diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java index 309b6be97f20b..5177d64cbdb06 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/PainlessDocGenerator.java @@ -23,9 +23,10 @@ import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.common.io.PathUtils; import org.elasticsearch.common.logging.ESLoggerFactory; -import org.elasticsearch.painless.Definition.Field; -import org.elasticsearch.painless.Definition.Method; -import org.elasticsearch.painless.Definition.Struct; +import org.elasticsearch.painless.lookup.PainlessLookup; +import org.elasticsearch.painless.lookup.PainlessField; +import org.elasticsearch.painless.lookup.PainlessMethod; +import org.elasticsearch.painless.lookup.PainlessClass; import java.io.IOException; import java.io.PrintStream; import java.lang.reflect.Modifier; @@ -44,15 +45,15 @@ import static org.elasticsearch.painless.spi.Whitelist.BASE_WHITELISTS; /** - * Generates an API reference from the method and type whitelists in {@link Definition}. + * Generates an API reference from the method and type whitelists in {@link PainlessLookup}. */ public class PainlessDocGenerator { - private static final Definition definition = new Definition(BASE_WHITELISTS); + private static final PainlessLookup PAINLESS_LOOKUP = new PainlessLookup(BASE_WHITELISTS); private static final Logger logger = ESLoggerFactory.getLogger(PainlessDocGenerator.class); - private static final Comparator FIELD_NAME = comparing(f -> f.name); - private static final Comparator METHOD_NAME = comparing(m -> m.name); - private static final Comparator NUMBER_OF_ARGS = comparing(m -> m.arguments.size()); + private static final Comparator FIELD_NAME = comparing(f -> f.name); + private static final Comparator METHOD_NAME = comparing(m -> m.name); + private static final Comparator NUMBER_OF_ARGS = comparing(m -> m.arguments.size()); public static void main(String[] args) throws IOException { Path apiRootPath = PathUtils.get(args[0]); @@ -67,8 +68,8 @@ public static void main(String[] args) throws IOException { Files.newOutputStream(indexPath, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE), false, StandardCharsets.UTF_8.name())) { emitGeneratedWarning(indexStream); - List structs = definition.getStructs().stream().sorted(comparing(t -> t.name)).collect(toList()); - for (Struct struct : structs) { + List structs = PAINLESS_LOOKUP.getStructs().stream().sorted(comparing(t -> t.name)).collect(toList()); + for (PainlessClass struct : structs) { if (struct.clazz.isPrimitive()) { // Primitives don't have methods to reference continue; @@ -93,13 +94,13 @@ public static void main(String[] args) throws IOException { typeStream.print(struct.name); typeStream.println("++::"); - Consumer documentField = field -> PainlessDocGenerator.documentField(typeStream, field); - Consumer documentMethod = method -> PainlessDocGenerator.documentMethod(typeStream, method); + Consumer documentField = field -> PainlessDocGenerator.documentField(typeStream, field); + Consumer documentMethod = method -> PainlessDocGenerator.documentMethod(typeStream, method); struct.staticMembers.values().stream().sorted(FIELD_NAME).forEach(documentField); struct.members.values().stream().sorted(FIELD_NAME).forEach(documentField); struct.staticMethods.values().stream().sorted(METHOD_NAME.thenComparing(NUMBER_OF_ARGS)).forEach(documentMethod); struct.constructors.values().stream().sorted(NUMBER_OF_ARGS).forEach(documentMethod); - Map inherited = new TreeMap<>(); + Map inherited = new TreeMap<>(); struct.methods.values().stream().sorted(METHOD_NAME.thenComparing(NUMBER_OF_ARGS)).forEach(method -> { if (method.owner == struct) { documentMethod(typeStream, method); @@ -111,7 +112,7 @@ public static void main(String[] args) throws IOException { if (false == inherited.isEmpty()) { typeStream.print("* Inherits methods from "); boolean first = true; - for (Struct inheritsFrom : inherited.values()) { + for (PainlessClass inheritsFrom : inherited.values()) { if (first) { first = false; } else { @@ -129,7 +130,7 @@ public static void main(String[] args) throws IOException { logger.info("Done writing [index.asciidoc]"); } - private static void documentField(PrintStream stream, Field field) { + private static void documentField(PrintStream stream, PainlessField field) { stream.print("** [["); emitAnchor(stream, field); stream.print("]]"); @@ -159,7 +160,7 @@ private static void documentField(PrintStream stream, Field field) { /** * Document a method. */ - private static void documentMethod(PrintStream stream, Method method) { + private static void documentMethod(PrintStream stream, PainlessMethod method) { stream.print("* ++[["); emitAnchor(stream, method); stream.print("]]"); @@ -201,17 +202,17 @@ private static void documentMethod(PrintStream stream, Method method) { } /** - * Anchor text for a {@link Struct}. + * Anchor text for a {@link PainlessClass}. */ - private static void emitAnchor(PrintStream stream, Struct struct) { + private static void emitAnchor(PrintStream stream, PainlessClass struct) { stream.print("painless-api-reference-"); stream.print(struct.name.replace('.', '-')); } /** - * Anchor text for a {@link Method}. + * Anchor text for a {@link PainlessMethod}. */ - private static void emitAnchor(PrintStream stream, Method method) { + private static void emitAnchor(PrintStream stream, PainlessMethod method) { emitAnchor(stream, method.owner); stream.print('-'); stream.print(methodName(method)); @@ -220,15 +221,15 @@ private static void emitAnchor(PrintStream stream, Method method) { } /** - * Anchor text for a {@link Field}. + * Anchor text for a {@link PainlessField}. */ - private static void emitAnchor(PrintStream stream, Field field) { + private static void emitAnchor(PrintStream stream, PainlessField field) { emitAnchor(stream, field.owner); stream.print('-'); stream.print(field.name); } - private static String methodName(Method method) { + private static String methodName(PainlessMethod method) { return method.name.equals("") ? method.owner.name : method.name; } @@ -237,17 +238,17 @@ private static String methodName(Method method) { an internal link with the text. */ private static void emitType(PrintStream stream, Class clazz) { - emitStruct(stream, definition.getPainlessStructFromJavaClass(clazz)); + emitStruct(stream, PAINLESS_LOOKUP.getPainlessStructFromJavaClass(clazz)); while ((clazz = clazz.getComponentType()) != null) { stream.print("[]"); } } /** - * Emit a {@link Struct}. If the {@linkplain Struct} is primitive or def this just emits the name of the struct. Otherwise this emits - * an internal link with the name. + * Emit a {@link PainlessClass}. If the {@linkplain PainlessClass} is primitive or def this just emits the name of the struct. + * Otherwise this emits an internal link with the name. */ - private static void emitStruct(PrintStream stream, Struct struct) { + private static void emitStruct(PrintStream stream, PainlessClass struct) { if (false == struct.clazz.isPrimitive() && false == struct.name.equals("def")) { stream.print("<<"); emitAnchor(stream, struct); @@ -260,11 +261,11 @@ private static void emitStruct(PrintStream stream, Struct struct) { } /** - * Emit an external link to Javadoc for a {@link Method}. + * Emit an external link to Javadoc for a {@link PainlessMethod}. * * @param root name of the root uri variable */ - private static void emitJavadocLink(PrintStream stream, String root, Method method) { + private static void emitJavadocLink(PrintStream stream, String root, PainlessMethod method) { stream.print("link:{"); stream.print(root); stream.print("-javadoc}/"); @@ -292,11 +293,11 @@ private static void emitJavadocLink(PrintStream stream, String root, Method meth } /** - * Emit an external link to Javadoc for a {@link Field}. + * Emit an external link to Javadoc for a {@link PainlessField}. * * @param root name of the root uri variable */ - private static void emitJavadocLink(PrintStream stream, String root, Field field) { + private static void emitJavadocLink(PrintStream stream, String root, PainlessField field) { stream.print("link:{"); stream.print(root); stream.print("-javadoc}/"); @@ -306,9 +307,9 @@ private static void emitJavadocLink(PrintStream stream, String root, Field field } /** - * Pick the javadoc root for a {@link Method}. + * Pick the javadoc root for a {@link PainlessMethod}. */ - private static String javadocRoot(Method method) { + private static String javadocRoot(PainlessMethod method) { if (method.augmentation != null) { return "painless"; } @@ -316,16 +317,16 @@ private static String javadocRoot(Method method) { } /** - * Pick the javadoc root for a {@link Field}. + * Pick the javadoc root for a {@link PainlessField}. */ - private static String javadocRoot(Field field) { + private static String javadocRoot(PainlessField field) { return javadocRoot(field.owner); } /** - * Pick the javadoc root for a {@link Struct}. + * Pick the javadoc root for a {@link PainlessClass}. */ - private static String javadocRoot(Struct struct) { + private static String javadocRoot(PainlessClass struct) { String classPackage = struct.clazz.getPackage().getName(); if (classPackage.startsWith("java")) { return "java8"; diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java index ea1d2275b3e8d..1a4770e560a7e 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/ScriptTestCase.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.lucene.ScorerAware; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.painless.antlr.Walker; +import org.elasticsearch.painless.lookup.PainlessLookup; import org.elasticsearch.painless.spi.Whitelist; import org.elasticsearch.script.ExecutableScript; import org.elasticsearch.script.ScriptContext; @@ -90,12 +91,12 @@ public Object exec(String script, Map vars, boolean picky) { public Object exec(String script, Map vars, Map compileParams, Scorer scorer, boolean picky) { // test for ambiguity errors before running the actual script if picky is true if (picky) { - Definition definition = new Definition(Whitelist.BASE_WHITELISTS); - ScriptClassInfo scriptClassInfo = new ScriptClassInfo(definition, GenericElasticsearchScript.class); + PainlessLookup painlessLookup = new PainlessLookup(Whitelist.BASE_WHITELISTS); + ScriptClassInfo scriptClassInfo = new ScriptClassInfo(painlessLookup, GenericElasticsearchScript.class); CompilerSettings pickySettings = new CompilerSettings(); pickySettings.setPicky(true); pickySettings.setRegexesEnabled(CompilerSettings.REGEX_ENABLED.get(scriptEngineSettings())); - Walker.buildPainlessTree(scriptClassInfo, new MainMethodReserved(), getTestName(), script, pickySettings, definition, null); + Walker.buildPainlessTree(scriptClassInfo, new MainMethodReserved(), getTestName(), script, pickySettings, painlessLookup, null); } // test actual script execution ExecutableScript.Factory factory = scriptEngine.compile(null, script, ExecutableScript.CONTEXT, compileParams); diff --git a/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java b/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java index fd8190aa2c2eb..3e9f724743faa 100644 --- a/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java +++ b/modules/lang-painless/src/test/java/org/elasticsearch/painless/node/NodeToStringTests.java @@ -20,12 +20,12 @@ package org.elasticsearch.painless.node; import org.elasticsearch.painless.CompilerSettings; -import org.elasticsearch.painless.Definition; -import org.elasticsearch.painless.Definition.Cast; -import org.elasticsearch.painless.Definition.Field; -import org.elasticsearch.painless.Definition.Method; -import org.elasticsearch.painless.Definition.MethodKey; -import org.elasticsearch.painless.Definition.Struct; +import org.elasticsearch.painless.lookup.PainlessLookup; +import org.elasticsearch.painless.lookup.PainlessCast; +import org.elasticsearch.painless.lookup.PainlessField; +import org.elasticsearch.painless.lookup.PainlessMethod; +import org.elasticsearch.painless.lookup.PainlessMethodKey; +import org.elasticsearch.painless.lookup.PainlessClass; import org.elasticsearch.painless.FeatureTest; import org.elasticsearch.painless.GenericElasticsearchScript; import org.elasticsearch.painless.Locals.Variable; @@ -48,7 +48,7 @@ * Tests {@link Object#toString} implementations on all extensions of {@link ANode}. */ public class NodeToStringTests extends ESTestCase { - private final Definition definition = new Definition(Whitelist.BASE_WHITELISTS); + private final PainlessLookup painlessLookup = new PainlessLookup(Whitelist.BASE_WHITELISTS); public void testEAssignment() { assertToString( @@ -161,12 +161,12 @@ public void testECapturingFunctionRef() { public void testECast() { Location l = new Location(getTestName(), 0); AExpression child = new EConstant(l, "test"); - Cast cast = Cast.standard(String.class, Integer.class, true); + PainlessCast cast = PainlessCast.standard(String.class, Integer.class, true); assertEquals("(ECast java.lang.Integer (EConstant String 'test'))", new ECast(l, child, cast).toString()); l = new Location(getTestName(), 1); child = new EBinary(l, Operation.ADD, new EConstant(l, "test"), new EConstant(l, 12)); - cast = Cast.standard(Integer.class, Boolean.class, true); + cast = PainlessCast.standard(Integer.class, Boolean.class, true); assertEquals("(ECast java.lang.Boolean (EBinary (EConstant String 'test') + (EConstant Integer 12)))", new ECast(l, child, cast).toString()); } @@ -403,15 +403,15 @@ public void testPSubBrace() { public void testPSubCallInvoke() { Location l = new Location(getTestName(), 0); - Struct c = definition.getPainlessStructFromJavaClass(Integer.class); - Method m = c.methods.get(new MethodKey("toString", 0)); + PainlessClass c = painlessLookup.getPainlessStructFromJavaClass(Integer.class); + PainlessMethod m = c.methods.get(new PainlessMethodKey("toString", 0)); PSubCallInvoke node = new PSubCallInvoke(l, m, null, emptyList()); node.prefix = new EVariable(l, "a"); assertEquals("(PSubCallInvoke (EVariable a) toString)", node.toString()); assertEquals("(PSubNullSafeCallInvoke (PSubCallInvoke (EVariable a) toString))", new PSubNullSafeCallInvoke(l, node).toString()); l = new Location(getTestName(), 1); - m = c.methods.get(new MethodKey("equals", 1)); + m = c.methods.get(new PainlessMethodKey("equals", 1)); node = new PSubCallInvoke(l, m, null, singletonList(new EVariable(l, "b"))); node.prefix = new EVariable(l, "a"); assertEquals("(PSubCallInvoke (EVariable a) equals (Args (EVariable b)))", node.toString()); @@ -458,8 +458,8 @@ public void testPSubDefField() { public void testPSubField() { Location l = new Location(getTestName(), 0); - Struct s = definition.getPainlessStructFromJavaClass(Boolean.class); - Field f = s.staticMembers.get("TRUE"); + PainlessClass s = painlessLookup.getPainlessStructFromJavaClass(Boolean.class); + PainlessField f = s.staticMembers.get("TRUE"); PSubField node = new PSubField(l, f); node.prefix = new EStatic(l, "Boolean"); assertEquals("(PSubField (EStatic Boolean) TRUE)", node.toString()); @@ -468,7 +468,7 @@ public void testPSubField() { public void testPSubListShortcut() { Location l = new Location(getTestName(), 0); - Struct s = definition.getPainlessStructFromJavaClass(List.class); + PainlessClass s = painlessLookup.getPainlessStructFromJavaClass(List.class); PSubListShortcut node = new PSubListShortcut(l, s, new EConstant(l, 1)); node.prefix = new EVariable(l, "a"); assertEquals("(PSubListShortcut (EVariable a) (EConstant Integer 1))", node.toString()); @@ -476,7 +476,7 @@ public void testPSubListShortcut() { new PSubNullSafeCallInvoke(l, node).toString()); l = new Location(getTestName(), 0); - s = definition.getPainlessStructFromJavaClass(List.class); + s = painlessLookup.getPainlessStructFromJavaClass(List.class); node = new PSubListShortcut(l, s, new EBinary(l, Operation.ADD, new EConstant(l, 1), new EConstant(l, 4))); node.prefix = new EVariable(l, "a"); assertEquals("(PSubListShortcut (EVariable a) (EBinary (EConstant Integer 1) + (EConstant Integer 4)))", node.toString()); @@ -484,7 +484,7 @@ public void testPSubListShortcut() { public void testPSubMapShortcut() { Location l = new Location(getTestName(), 0); - Struct s = definition.getPainlessStructFromJavaClass(Map.class); + PainlessClass s = painlessLookup.getPainlessStructFromJavaClass(Map.class); PSubMapShortcut node = new PSubMapShortcut(l, s, new EConstant(l, "cat")); node.prefix = new EVariable(l, "a"); assertEquals("(PSubMapShortcut (EVariable a) (EConstant String 'cat'))", node.toString()); @@ -492,7 +492,7 @@ public void testPSubMapShortcut() { new PSubNullSafeCallInvoke(l, node).toString()); l = new Location(getTestName(), 1); - s = definition.getPainlessStructFromJavaClass(Map.class); + s = painlessLookup.getPainlessStructFromJavaClass(Map.class); node = new PSubMapShortcut(l, s, new EBinary(l, Operation.ADD, new EConstant(l, 1), new EConstant(l, 4))); node.prefix = new EVariable(l, "a"); assertEquals("(PSubMapShortcut (EVariable a) (EBinary (EConstant Integer 1) + (EConstant Integer 4)))", node.toString()); @@ -500,9 +500,9 @@ public void testPSubMapShortcut() { public void testPSubShortcut() { Location l = new Location(getTestName(), 0); - Struct s = definition.getPainlessStructFromJavaClass(FeatureTest.class); - Method getter = s.methods.get(new MethodKey("getX", 0)); - Method setter = s.methods.get(new MethodKey("setX", 1)); + PainlessClass s = painlessLookup.getPainlessStructFromJavaClass(FeatureTest.class); + PainlessMethod getter = s.methods.get(new PainlessMethodKey("getX", 0)); + PainlessMethod setter = s.methods.get(new PainlessMethodKey("setX", 1)); PSubShortcut node = new PSubShortcut(l, "x", FeatureTest.class.getName(), getter, setter); node.prefix = new EVariable(l, "a"); assertEquals("(PSubShortcut (EVariable a) x)", node.toString()); @@ -900,12 +900,12 @@ private void assertToString(String expected, String code) { } private SSource walk(String code) { - ScriptClassInfo scriptClassInfo = new ScriptClassInfo(definition, GenericElasticsearchScript.class); + ScriptClassInfo scriptClassInfo = new ScriptClassInfo(painlessLookup, GenericElasticsearchScript.class); CompilerSettings compilerSettings = new CompilerSettings(); compilerSettings.setRegexesEnabled(true); try { return Walker.buildPainlessTree( - scriptClassInfo, new MainMethodReserved(), getTestName(), code, compilerSettings, definition, null); + scriptClassInfo, new MainMethodReserved(), getTestName(), code, compilerSettings, painlessLookup, null); } catch (Exception e) { throw new AssertionError("Failed to compile: " + code, e); } From a000a7e6d8e2586132cf6845c633b1f3e866297a Mon Sep 17 00:00:00 2001 From: Russ Cam Date: Fri, 6 Jul 2018 15:36:31 +1000 Subject: [PATCH 02/50] Remove link to oss-MSI (#31844) This commit removes the link to an oss-MSI; there is only one version of the MSI, which includes X-Pack. (cherry picked from commit d2e5db8a806ec8a25162f79db5209aceed4f30f7) --- docs/reference/setup/install/windows.asciidoc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/reference/setup/install/windows.asciidoc b/docs/reference/setup/install/windows.asciidoc index 1535e5415e4a4..f5e248598ca9f 100644 --- a/docs/reference/setup/install/windows.asciidoc +++ b/docs/reference/setup/install/windows.asciidoc @@ -34,10 +34,6 @@ ifeval::["{release-state}"!="unreleased"] Download the `.msi` package for Elasticsearch v{version} from https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-{version}.msi -Alternatively, you can download the following package, which contains only -features that are available under the Apache 2.0 license: -https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-oss-{version}.msi - endif::[] [[install-msi-gui]] From 4a005225bb6f21fb1aecfd81dbee285965dc2644 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Sun, 8 Jul 2018 11:03:56 -0400 Subject: [PATCH 03/50] HLREST: Add x-pack-info API (#31870) This is the first x-pack API we're adding to the high level REST client so there is a lot to talk about here! = Open source The *client* for these APIs is open source. We're taking the previously Elastic licensed files used for the `Request` and `Response` objects and relicensing them under the Apache 2 license. The implementation of these features is staying under the Elastic license. This lines up with how the rest of the Elasticsearch language clients work. = Location of the new files We're moving all of the `Request` and `Response` objects that we're relicensing to the `x-pack/protocol` directory. We're adding a copy of the Apache 2 license to the root fo the `x-pack/protocol` directory to line up with the language in the root `LICENSE.txt` file. All files in this directory will have the Apache 2 license header as well. We don't want there to be any confusion. Even though the files are under the `x-pack` directory, they are Apache 2 licensed. We chose this particular directory layout because it keeps the X-Pack stuff together and easier to think about. = Location of the API in the REST client We've been following the layout of the rest-api-spec files for other APIs and we plan to do this for the X-Pack APIs with one exception: we're dropping the `xpack` from the name of most of the APIs. So `xpack.graph.explore` will become `graph().explore()` and `xpack.license.get` will become `license().get()`. `xpack.info` and `xpack.usage` are special here though because they don't belong to any proper category. For now I'm just calling `xpack.info` `xPackInfo()` and intend to call usage `xPackUsage` though I'm not convinced that this is the final name for them. But it does get us started. = Jars, jars everywhere! This change makes the `xpack:protocol` project a `compile` scoped dependency of the `x-pack:plugin:core` and `client:rest-high-level` projects. I intend to keep it a compile scoped dependency of `x-pack:plugin:core` but I intend to bundle the contents of the protocol jar into the `client:rest-high-level` jar in a follow up. This change has grown large enough at this point. In that followup I'll address javadoc issues as well. = Breaking-Java This breaks that transport client by a few classes around. We've traditionally been ok with doing this to the transport client. --- client/rest-high-level/build.gradle | 1 + .../client/RequestConverters.java | 16 + .../client/RestHighLevelClient.java | 39 +- .../elasticsearch/client/PingAndInfoIT.java | 52 +- .../client/RequestConvertersTests.java | 33 ++ .../MiscellaneousDocumentationIT.java | 63 +++ .../miscellaneous/x-pack-info.asciidoc | 64 +++ .../high-level/supported-apis.asciidoc | 3 +- .../common/xcontent/ObjectParser.java | 1 + x-pack/build.gradle | 23 +- x-pack/plugin/core/build.gradle | 1 + .../org/elasticsearch/license/License.java | 44 +- .../elasticsearch/license/LicenseService.java | 4 +- .../license/XPackInfoResponse.java | 298 ----------- .../elasticsearch/xpack/core/XPackClient.java | 4 +- .../core/action/TransportXPackInfoAction.java | 13 +- .../xpack/core/action/XPackInfoAction.java | 3 +- .../core/action/XPackInfoRequestBuilder.java | 3 +- .../core/rest/action/RestXPackInfoAction.java | 41 +- .../action/TransportXPackInfoActionTests.java | 8 +- .../ml/datafeed/MlRemoteLicenseChecker.java | 9 +- .../datafeed/MlRemoteLicenseCheckerTests.java | 18 +- .../esnative/tool/SetupPasswordToolTests.java | 6 +- x-pack/protocol/LICENSE.txt | 202 +++++++ x-pack/protocol/build.gradle | 29 + .../protocol/license/LicenseStatus.java | 67 +++ .../protocol/license/package-info.java | 24 + .../protocol/security/package-info.java | 24 + .../protocol/watcher/package-info.java | 24 + .../protocol/xpack}/XPackInfoRequest.java | 24 +- .../protocol/xpack/XPackInfoResponse.java | 500 ++++++++++++++++++ .../protocol/xpack/package-info.java | 23 + .../protocol/license/LicenseStatusTests.java | 30 ++ .../xpack/XPackInfoResponseTests.java | 160 ++++++ 34 files changed, 1429 insertions(+), 425 deletions(-) create mode 100644 docs/java-rest/high-level/miscellaneous/x-pack-info.asciidoc delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackInfoResponse.java create mode 100644 x-pack/protocol/LICENSE.txt create mode 100644 x-pack/protocol/build.gradle create mode 100644 x-pack/protocol/src/main/java/org/elasticsearch/protocol/license/LicenseStatus.java create mode 100644 x-pack/protocol/src/main/java/org/elasticsearch/protocol/license/package-info.java create mode 100644 x-pack/protocol/src/main/java/org/elasticsearch/protocol/security/package-info.java create mode 100644 x-pack/protocol/src/main/java/org/elasticsearch/protocol/watcher/package-info.java rename x-pack/{plugin/core/src/main/java/org/elasticsearch/xpack/core/action => protocol/src/main/java/org/elasticsearch/protocol/xpack}/XPackInfoRequest.java (71%) create mode 100644 x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoResponse.java create mode 100644 x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/package-info.java create mode 100644 x-pack/protocol/src/test/java/org/elasticsearch/protocol/license/LicenseStatusTests.java create mode 100644 x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/XPackInfoResponseTests.java diff --git a/client/rest-high-level/build.gradle b/client/rest-high-level/build.gradle index 222de9608aeb9..05b1cb25ed5d0 100644 --- a/client/rest-high-level/build.gradle +++ b/client/rest-high-level/build.gradle @@ -41,6 +41,7 @@ dependencies { compile "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}" compile "org.elasticsearch.plugin:rank-eval-client:${version}" compile "org.elasticsearch.plugin:lang-mustache-client:${version}" + compile project(':x-pack:protocol') // TODO bundle into the jar testCompile "org.elasticsearch.client:test:${version}" testCompile "org.elasticsearch.test:framework:${version}" diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java index 6d31dcd40d416..9396baa9420d9 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java @@ -103,6 +103,7 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.index.VersionType; import org.elasticsearch.index.rankeval.RankEvalRequest; +import org.elasticsearch.protocol.xpack.XPackInfoRequest; import org.elasticsearch.rest.action.search.RestSearchAction; import org.elasticsearch.script.mustache.MultiSearchTemplateRequest; import org.elasticsearch.script.mustache.SearchTemplateRequest; @@ -114,8 +115,10 @@ import java.net.URI; import java.net.URISyntaxException; import java.nio.charset.Charset; +import java.util.EnumSet; import java.util.Locale; import java.util.StringJoiner; +import java.util.stream.Collectors; final class RequestConverters { static final XContentType REQUEST_BODY_CONTENT_TYPE = XContentType.JSON; @@ -1055,6 +1058,19 @@ static Request deleteScript(DeleteStoredScriptRequest deleteStoredScriptRequest) return request; } + static Request xPackInfo(XPackInfoRequest infoRequest) { + Request request = new Request(HttpGet.METHOD_NAME, "/_xpack"); + if (false == infoRequest.isVerbose()) { + request.addParameter("human", "false"); + } + if (false == infoRequest.getCategories().equals(EnumSet.allOf(XPackInfoRequest.Category.class))) { + request.addParameter("categories", infoRequest.getCategories().stream() + .map(c -> c.toString().toLowerCase(Locale.ROOT)) + .collect(Collectors.joining(","))); + } + return request; + } + private static HttpEntity createEntity(ToXContent toXContent, XContentType xContentType) throws IOException { BytesRef source = XContentHelper.toXContent(toXContent, xContentType, false).toBytesRef(); return new ByteArrayEntity(source.bytes, source.offset, source.length, createContentType(xContentType)); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java index 40125b92512f7..bff43bdd6e3b2 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java @@ -66,6 +66,8 @@ import org.elasticsearch.index.rankeval.RankEvalRequest; import org.elasticsearch.index.rankeval.RankEvalResponse; import org.elasticsearch.plugins.spi.NamedXContentProvider; +import org.elasticsearch.protocol.xpack.XPackInfoRequest; +import org.elasticsearch.protocol.xpack.XPackInfoResponse; import org.elasticsearch.rest.BytesRestResponse; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.script.mustache.MultiSearchTemplateRequest; @@ -979,7 +981,6 @@ public final void rankEvalAsync(RankEvalRequest rankEvalRequest, RequestOptions emptySet()); } - /** * Executes a request using the Multi Search Template API. * @@ -989,9 +990,9 @@ public final void rankEvalAsync(RankEvalRequest rankEvalRequest, RequestOptions public final MultiSearchTemplateResponse multiSearchTemplate(MultiSearchTemplateRequest multiSearchTemplateRequest, RequestOptions options) throws IOException { return performRequestAndParseEntity(multiSearchTemplateRequest, RequestConverters::multiSearchTemplate, - options, MultiSearchTemplateResponse::fromXContext, emptySet()); - } - + options, MultiSearchTemplateResponse::fromXContext, emptySet()); + } + /** * Asynchronously executes a request using the Multi Search Template API * @@ -1003,7 +1004,7 @@ public final void multiSearchTemplateAsync(MultiSearchTemplateRequest multiSearc ActionListener listener) { performRequestAsyncAndParseEntity(multiSearchTemplateRequest, RequestConverters::multiSearchTemplate, options, MultiSearchTemplateResponse::fromXContext, listener, emptySet()); - } + } /** * Asynchronously executes a request using the Ranking Evaluation API. @@ -1103,6 +1104,34 @@ public final void fieldCapsAsync(FieldCapabilitiesRequest fieldCapabilitiesReque FieldCapabilitiesResponse::fromXContent, listener, emptySet()); } + /** + * Fetch information about X-Pack from the cluster if it is installed. + * See + * the docs for more. + * @param request the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public XPackInfoResponse xPackInfo(XPackInfoRequest request, RequestOptions options) throws IOException { + return performRequestAndParseEntity(request, RequestConverters::xPackInfo, options, + XPackInfoResponse::fromXContent, emptySet()); + } + + /** + * Fetch information about X-Pack from the cluster if it is installed. + * See + * the docs for more. + * @param request the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public void xPackInfoAsync(XPackInfoRequest request, RequestOptions options, + ActionListener listener) { + performRequestAsyncAndParseEntity(request, RequestConverters::xPackInfo, options, + XPackInfoResponse::fromXContent, listener, emptySet()); + } + @Deprecated protected final Resp performRequestAndParseEntity(Req request, CheckedFunction requestConverter, diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/PingAndInfoIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/PingAndInfoIT.java index 057ea49f9a969..04bac628f4930 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/PingAndInfoIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/PingAndInfoIT.java @@ -21,8 +21,13 @@ import org.apache.http.client.methods.HttpGet; import org.elasticsearch.action.main.MainResponse; +import org.elasticsearch.protocol.license.LicenseStatus; +import org.elasticsearch.protocol.xpack.XPackInfoRequest; +import org.elasticsearch.protocol.xpack.XPackInfoResponse; +import org.elasticsearch.protocol.xpack.XPackInfoResponse.FeatureSetsInfo.FeatureSet; import java.io.IOException; +import java.util.EnumSet; import java.util.Map; public class PingAndInfoIT extends ESRestHighLevelClientTestCase { @@ -31,7 +36,6 @@ public void testPing() throws IOException { assertTrue(highLevelClient().ping(RequestOptions.DEFAULT)); } - @SuppressWarnings("unchecked") public void testInfo() throws IOException { MainResponse info = highLevelClient().info(RequestOptions.DEFAULT); // compare with what the low level client outputs @@ -41,6 +45,7 @@ public void testInfo() throws IOException { // only check node name existence, might be a different one from what was hit by low level client in multi-node cluster assertNotNull(info.getNodeName()); + @SuppressWarnings("unchecked") Map versionMap = (Map) infoAsMap.get("version"); assertEquals(versionMap.get("build_flavor"), info.getBuild().flavor().displayName()); assertEquals(versionMap.get("build_type"), info.getBuild().type().displayName()); @@ -51,4 +56,49 @@ public void testInfo() throws IOException { assertEquals(versionMap.get("lucene_version"), info.getVersion().luceneVersion.toString()); } + public void testXPackInfo() throws IOException { + XPackInfoRequest request = new XPackInfoRequest(); + request.setCategories(EnumSet.allOf(XPackInfoRequest.Category.class)); + request.setVerbose(true); + XPackInfoResponse info = highLevelClient().xPackInfo(request, RequestOptions.DEFAULT); + + MainResponse mainResponse = highLevelClient().info(RequestOptions.DEFAULT); + + assertEquals(mainResponse.getBuild().shortHash(), info.getBuildInfo().getHash()); + + assertEquals("basic", info.getLicenseInfo().getType()); + assertEquals("basic", info.getLicenseInfo().getMode()); + assertEquals(LicenseStatus.ACTIVE, info.getLicenseInfo().getStatus()); + + FeatureSet graph = info.getFeatureSetsInfo().getFeatureSets().get("graph"); + assertNotNull(graph.description()); + assertFalse(graph.available()); + assertTrue(graph.enabled()); + assertNull(graph.nativeCodeInfo()); + FeatureSet monitoring = info.getFeatureSetsInfo().getFeatureSets().get("monitoring"); + assertNotNull(monitoring.description()); + assertTrue(monitoring.available()); + assertTrue(monitoring.enabled()); + assertNull(monitoring.nativeCodeInfo()); + FeatureSet ml = info.getFeatureSetsInfo().getFeatureSets().get("ml"); + assertNotNull(ml.description()); + assertFalse(ml.available()); + assertTrue(ml.enabled()); + assertEquals(mainResponse.getVersion().toString(), + ml.nativeCodeInfo().get("version").toString().replace("-SNAPSHOT", "")); + } + + public void testXPackInfoEmptyRequest() throws IOException { + XPackInfoResponse info = highLevelClient().xPackInfo(new XPackInfoRequest(), RequestOptions.DEFAULT); + + /* + * The default in the transport client is non-verbose and returning + * no categories which is the opposite of the default when you use + * the API over REST. We don't want to break the transport client + * even though it doesn't feel like a good default. + */ + assertNull(info.getBuildInfo()); + assertNull(info.getLicenseInfo()); + assertNull(info.getFeatureSetsInfo()); + } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java index c025507108325..f18c102217bfe 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java @@ -122,6 +122,7 @@ import org.elasticsearch.index.rankeval.RankEvalSpec; import org.elasticsearch.index.rankeval.RatedRequest; import org.elasticsearch.index.rankeval.RestRankEvalAction; +import org.elasticsearch.protocol.xpack.XPackInfoRequest; import org.elasticsearch.repositories.fs.FsRepository; import org.elasticsearch.rest.action.search.RestSearchAction; import org.elasticsearch.script.ScriptType; @@ -149,6 +150,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -2437,6 +2439,37 @@ public void testEnforceSameContentType() { + "previous requests have content-type [" + xContentType + "]", exception.getMessage()); } + public void testXPackInfo() { + XPackInfoRequest infoRequest = new XPackInfoRequest(); + Map expectedParams = new HashMap<>(); + infoRequest.setVerbose(randomBoolean()); + if (false == infoRequest.isVerbose()) { + expectedParams.put("human", "false"); + } + int option = between(0, 2); + switch (option) { + case 0: + infoRequest.setCategories(EnumSet.allOf(XPackInfoRequest.Category.class)); + break; + case 1: + infoRequest.setCategories(EnumSet.of(XPackInfoRequest.Category.FEATURES)); + expectedParams.put("categories", "features"); + break; + case 2: + infoRequest.setCategories(EnumSet.of(XPackInfoRequest.Category.FEATURES, XPackInfoRequest.Category.BUILD)); + expectedParams.put("categories", "build,features"); + break; + default: + throw new IllegalArgumentException("invalid option [" + option + "]"); + } + + Request request = RequestConverters.xPackInfo(infoRequest); + assertEquals(HttpGet.METHOD_NAME, request.getMethod()); + assertEquals("/_xpack", request.getEndpoint()); + assertNull(request.getEntity()); + assertEquals(expectedParams, request.getParameters()); + } + /** * Randomize the {@link FetchSourceContext} request parameters. */ diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MiscellaneousDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MiscellaneousDocumentationIT.java index 0c1c446961bdf..1729e6b22aa5b 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MiscellaneousDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MiscellaneousDocumentationIT.java @@ -21,6 +21,8 @@ import org.elasticsearch.Build; import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.LatchedActionListener; import org.elasticsearch.action.main.MainResponse; import org.elasticsearch.client.ESRestHighLevelClientTestCase; import org.elasticsearch.client.RequestOptions; @@ -28,8 +30,16 @@ import org.elasticsearch.client.RestHighLevelClient; import org.apache.http.HttpHost; import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.protocol.xpack.XPackInfoRequest; +import org.elasticsearch.protocol.xpack.XPackInfoResponse; +import org.elasticsearch.protocol.xpack.XPackInfoResponse.BuildInfo; +import org.elasticsearch.protocol.xpack.XPackInfoResponse.FeatureSetsInfo; +import org.elasticsearch.protocol.xpack.XPackInfoResponse.LicenseInfo; import java.io.IOException; +import java.util.EnumSet; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; /** * Documentation for miscellaneous APIs in the high level java client. @@ -67,6 +77,59 @@ public void testPing() throws IOException { assertTrue(response); } + public void testXPackInfo() throws Exception { + RestHighLevelClient client = highLevelClient(); + { + //tag::x-pack-info-execute + XPackInfoRequest request = new XPackInfoRequest(); + request.setVerbose(true); // <1> + request.setCategories(EnumSet.of( // <2> + XPackInfoRequest.Category.BUILD, + XPackInfoRequest.Category.LICENSE, + XPackInfoRequest.Category.FEATURES)); + XPackInfoResponse response = client.xPackInfo(request, RequestOptions.DEFAULT); + //end::x-pack-info-execute + + //tag::x-pack-info-response + BuildInfo build = response.getBuildInfo(); // <1> + LicenseInfo license = response.getLicenseInfo(); // <2> + assertEquals(XPackInfoResponse.BASIC_SELF_GENERATED_LICENSE_EXPIRATION_MILLIS, + license.getExpiryDate()); // <3> + FeatureSetsInfo features = response.getFeatureSetsInfo(); // <4> + //end::x-pack-info-response + + assertNotNull(response.getBuildInfo()); + assertNotNull(response.getLicenseInfo()); + assertNotNull(response.getFeatureSetsInfo()); + } + { + XPackInfoRequest request = new XPackInfoRequest(); + // tag::x-pack-info-execute-listener + ActionListener listener = new ActionListener() { + @Override + public void onResponse(XPackInfoResponse indexResponse) { + // <1> + } + + @Override + public void onFailure(Exception e) { + // <2> + } + }; + // end::x-pack-info-execute-listener + + // Replace the empty listener by a blocking listener in test + final CountDownLatch latch = new CountDownLatch(1); + listener = new LatchedActionListener<>(listener, latch); + + // tag::x-pack-info-execute-async + client.xPackInfoAsync(request, RequestOptions.DEFAULT, listener); // <1> + // end::x-pack-info-execute-async + + assertTrue(latch.await(30L, TimeUnit.SECONDS)); + } + } + public void testInitializationFromClientBuilder() throws IOException { //tag::rest-high-level-client-init RestHighLevelClient client = new RestHighLevelClient( diff --git a/docs/java-rest/high-level/miscellaneous/x-pack-info.asciidoc b/docs/java-rest/high-level/miscellaneous/x-pack-info.asciidoc new file mode 100644 index 0000000000000..f877ed720db69 --- /dev/null +++ b/docs/java-rest/high-level/miscellaneous/x-pack-info.asciidoc @@ -0,0 +1,64 @@ +[[java-rest-high-x-pack-info]] +=== X-Pack Info API + +[[java-rest-high-x-pack-info-execution]] +==== Execution + +General information about the installed {xpack} features can be retrieved +using the `xPackInfo()` method: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/MiscellaneousDocumentationIT.java[x-pack-info-execute] +-------------------------------------------------- +<1> Enable verbose mode. The default is `false` but `true` will return +more information. +<2> Set the categories of information to retrieve. The the default is to +return no information which is useful for checking if {xpack} is installed +but not much else. + +[[java-rest-high-x-pack-info-response]] +==== Response + +The returned `XPackInfoResponse` can contain `BuildInfo`, `LicenseInfo`, +and `FeatureSetsInfo` depending on the categories requested. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/MiscellaneousDocumentationIT.java[x-pack-info-response] +-------------------------------------------------- +<1> `BuildInfo` contains the commit hash from which Elasticsearch was +built and the timestamp that the x-pack module was created. +<2> `LicenseInfo` contains the type of license that the cluster is using +and its expiration date. +<3> Basic licenses do not expire and will return this constant. +<4> `FeatureSetsInfo` contains a `Map` from the name of a feature to +information about a feature like whether or not it is available under +the current license. + +[[java-rest-high-x-pack-info-async]] +==== Asynchronous Execution + +This request can be executed asynchronously: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/MiscellaneousDocumentationIT.java[x-pack-info-execute-async] +-------------------------------------------------- +<1> The `XPackInfoRequest` to execute and the `ActionListener` to use when +the execution completes + +The asynchronous method does not block and returns immediately. Once it is +completed the `ActionListener` is called back using the `onResponse` method +if the execution successfully completed or using the `onFailure` method if +it failed. + +A typical listener for `XPackInfoResponse` looks like: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/MiscellaneousDocumentationIT.java[x-pack-info-execute-listener] +-------------------------------------------------- +<1> Called when the execution is successfully completed. The response is +provided as an argument +<2> Called in case of failure. The raised exception is provided as an argument diff --git a/docs/java-rest/high-level/supported-apis.asciidoc b/docs/java-rest/high-level/supported-apis.asciidoc index 9a261b9ecdf25..447c984adf380 100644 --- a/docs/java-rest/high-level/supported-apis.asciidoc +++ b/docs/java-rest/high-level/supported-apis.asciidoc @@ -53,9 +53,11 @@ The Java High Level REST Client supports the following Miscellaneous APIs: * <> * <> +* <> include::miscellaneous/main.asciidoc[] include::miscellaneous/ping.asciidoc[] +include::miscellaneous/x-pack-info.asciidoc[] == Indices APIs @@ -179,4 +181,3 @@ The Java High Level REST Client supports the following Scripts APIs: include::script/get_script.asciidoc[] include::script/delete_script.asciidoc[] - diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java index dfcc4271b922e..d0cc929b56d24 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ObjectParser.java @@ -411,6 +411,7 @@ public enum ValueType { INT_ARRAY(START_ARRAY, VALUE_NUMBER, VALUE_STRING), BOOLEAN_ARRAY(START_ARRAY, VALUE_BOOLEAN), OBJECT(START_OBJECT), + OBJECT_OR_NULL(START_OBJECT, VALUE_NULL), OBJECT_ARRAY(START_OBJECT, START_ARRAY), OBJECT_OR_BOOLEAN(START_OBJECT, VALUE_BOOLEAN), OBJECT_OR_STRING(START_OBJECT, VALUE_STRING), diff --git a/x-pack/build.gradle b/x-pack/build.gradle index ca06b0763ed26..01ce465fc0938 100644 --- a/x-pack/build.gradle +++ b/x-pack/build.gradle @@ -12,16 +12,23 @@ subprojects { // helper method to find the path to a module ext.xpackModule = { String moduleName -> xpackProject("plugin:${moduleName}").path } - ext.licenseName = 'Elastic License' - ext.licenseUrl = ext.elasticLicenseUrl - - project.ext.licenseFile = rootProject.file('licenses/ELASTIC-LICENSE.txt') - project.ext.noticeFile = xpackRootProject.file('NOTICE.txt') - plugins.withType(PluginBuildPlugin).whenPluginAdded { project.esplugin.licenseFile = rootProject.file('licenses/ELASTIC-LICENSE.txt') project.esplugin.noticeFile = xpackRootProject.file('NOTICE.txt') } + + if (project.name != 'protocol') { + tasks.withType(LicenseHeadersTask.class) { + approvedLicenses = ['Elastic License', 'Generated'] + additionalLicense 'ELAST', 'Elastic License', 'Licensed under the Elastic License' + } + + ext.licenseName = 'Elastic License' + ext.licenseUrl = ext.elasticLicenseUrl + + project.ext.licenseFile = rootProject.file('licenses/ELASTIC-LICENSE.txt') + project.ext.noticeFile = xpackRootProject.file('NOTICE.txt') + } } File checkstyleSuppressions = file('dev-tools/checkstyle_suppressions.xml') @@ -34,10 +41,6 @@ subprojects { ] } - tasks.withType(LicenseHeadersTask.class) { - approvedLicenses = ['Elastic License', 'Generated'] - additionalLicense 'ELAST', 'Elastic License', 'Licensed under the Elastic License' - } ext.projectSubstitutions += [ "org.elasticsearch.plugin:x-pack-core:${version}": xpackModule('core')] ext.projectSubstitutions += [ "org.elasticsearch.plugin:x-pack-deprecation:${version}": xpackModule('deprecation')] ext.projectSubstitutions += [ "org.elasticsearch.plugin:x-pack-graph:${version}": xpackModule('graph')] diff --git a/x-pack/plugin/core/build.gradle b/x-pack/plugin/core/build.gradle index 4bb0e0ffc031b..5db149bc6774e 100644 --- a/x-pack/plugin/core/build.gradle +++ b/x-pack/plugin/core/build.gradle @@ -25,6 +25,7 @@ dependencyLicenses { dependencies { compileOnly "org.elasticsearch:elasticsearch:${version}" + compile project(':x-pack:protocol') compile "org.apache.httpcomponents:httpclient:${versions.httpclient}" compile "org.apache.httpcomponents:httpcore:${versions.httpcore}" compile "org.apache.httpcomponents:httpcore-nio:${versions.httpcore}" diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/License.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/License.java index e9be9fec66970..78a66c99323d0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/License.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/License.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.protocol.license.LicenseStatus; /** * Data structure for license. Use {@link Builder} to build a license. @@ -267,14 +268,14 @@ public synchronized void removeOperationModeFileWatcher() { /** * @return the current license's status */ - public Status status() { + public LicenseStatus status() { long now = System.currentTimeMillis(); if (issueDate > now) { - return Status.INVALID; + return LicenseStatus.INVALID; } else if (expiryDate < now) { - return Status.EXPIRED; + return LicenseStatus.EXPIRED; } - return Status.ACTIVE; + return LicenseStatus.ACTIVE; } private void validate() { @@ -767,41 +768,6 @@ public Builder validate() { } } - public enum Status { - - ACTIVE("active"), - INVALID("invalid"), - EXPIRED("expired"); - - private final String label; - - Status(String label) { - this.label = label; - } - - public String label() { - return label; - } - - public void writeTo(StreamOutput out) throws IOException { - out.writeString(label); - } - - public static Status readFrom(StreamInput in) throws IOException { - String value = in.readString(); - switch (value) { - case "active": - return ACTIVE; - case "invalid": - return INVALID; - case "expired": - return EXPIRED; - default: - throw new IllegalArgumentException("unknown license status [" + value + "]"); - } - } - } - /** * Returns true iff the license is a production licnese */ diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseService.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseService.java index 40c694cedb764..a39e9f412d767 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseService.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/LicenseService.java @@ -27,6 +27,7 @@ import org.elasticsearch.discovery.DiscoveryModule; import org.elasticsearch.env.Environment; import org.elasticsearch.gateway.GatewayService; +import org.elasticsearch.protocol.xpack.XPackInfoResponse; import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.XPackSettings; @@ -72,7 +73,8 @@ public class LicenseService extends AbstractLifecycleComponent implements Cluste */ static final TimeValue GRACE_PERIOD_DURATION = days(7); - public static final long BASIC_SELF_GENERATED_LICENSE_EXPIRATION_MILLIS = Long.MAX_VALUE - days(365).millis(); + public static final long BASIC_SELF_GENERATED_LICENSE_EXPIRATION_MILLIS = + XPackInfoResponse.BASIC_SELF_GENERATED_LICENSE_EXPIRATION_MILLIS; private final ClusterService clusterService; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackInfoResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackInfoResponse.java deleted file mode 100644 index 4d5c90ada4960..0000000000000 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackInfoResponse.java +++ /dev/null @@ -1,298 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -package org.elasticsearch.license; - -import org.elasticsearch.Version; -import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.xcontent.ToXContentObject; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.xpack.core.XPackBuild; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -public class XPackInfoResponse extends ActionResponse { - - @Nullable private BuildInfo buildInfo; - @Nullable private LicenseInfo licenseInfo; - @Nullable private FeatureSetsInfo featureSetsInfo; - - public XPackInfoResponse() {} - - public XPackInfoResponse(@Nullable BuildInfo buildInfo, @Nullable LicenseInfo licenseInfo, @Nullable FeatureSetsInfo featureSetsInfo) { - this.buildInfo = buildInfo; - this.licenseInfo = licenseInfo; - this.featureSetsInfo = featureSetsInfo; - } - - /** - * @return The build info (incl. build hash and timestamp) - */ - public BuildInfo getBuildInfo() { - return buildInfo; - } - - /** - * @return The current license info (incl. UID, type/mode. status and expiry date). May return {@code null} when no - * license is currently installed. - */ - public LicenseInfo getLicenseInfo() { - return licenseInfo; - } - - /** - * @return The current status of the feature sets in X-Pack. Feature sets describe the features available/enabled in X-Pack. - */ - public FeatureSetsInfo getFeatureSetsInfo() { - return featureSetsInfo; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - super.writeTo(out); - out.writeOptionalWriteable(buildInfo); - out.writeOptionalWriteable(licenseInfo); - out.writeOptionalWriteable(featureSetsInfo); - } - - @Override - public void readFrom(StreamInput in) throws IOException { - this.buildInfo = in.readOptionalWriteable(BuildInfo::new); - this.licenseInfo = in.readOptionalWriteable(LicenseInfo::new); - this.featureSetsInfo = in.readOptionalWriteable(FeatureSetsInfo::new); - } - - public static class LicenseInfo implements ToXContentObject, Writeable { - - private final String uid; - private final String type; - private final String mode; - private final long expiryDate; - private final License.Status status; - - public LicenseInfo(License license) { - this(license.uid(), license.type(), license.operationMode().name().toLowerCase(Locale.ROOT), - license.status(), license.expiryDate()); - } - - public LicenseInfo(StreamInput in) throws IOException { - this(in.readString(), in.readString(), in.readString(), License.Status.readFrom(in), in.readLong()); - } - - public LicenseInfo(String uid, String type, String mode, License.Status status, long expiryDate) { - this.uid = uid; - this.type = type; - this.mode = mode; - this.status = status; - this.expiryDate = expiryDate; - } - - public String getUid() { - return uid; - } - - public String getType() { - return type; - } - - public String getMode() { - return mode; - } - - public long getExpiryDate() { - return expiryDate; - } - - public License.Status getStatus() { - return status; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject() - .field("uid", uid) - .field("type", type) - .field("mode", mode) - .field("status", status.label()); - if (expiryDate != LicenseService.BASIC_SELF_GENERATED_LICENSE_EXPIRATION_MILLIS) { - builder.timeField("expiry_date_in_millis", "expiry_date", expiryDate); - } - return builder.endObject(); - } - - public void writeTo(StreamOutput out) throws IOException { - out.writeString(uid); - out.writeString(type); - out.writeString(mode); - status.writeTo(out); - out.writeLong(expiryDate); - } - } - - public static class BuildInfo implements ToXContentObject, Writeable { - - private final String hash; - private final String timestamp; - - public BuildInfo(XPackBuild build) { - this(build.shortHash(), build.date()); - } - - public BuildInfo(StreamInput input) throws IOException { - this(input.readString(), input.readString()); - } - - public BuildInfo(String hash, String timestamp) { - this.hash = hash; - this.timestamp = timestamp; - } - - public String getHash() { - return hash; - } - - public String getTimestamp() { - return timestamp; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - return builder.startObject() - .field("hash", hash) - .field("date", timestamp) - .endObject(); - } - - public void writeTo(StreamOutput output) throws IOException { - output.writeString(hash); - output.writeString(timestamp); - } - } - - public static class FeatureSetsInfo implements ToXContentObject, Writeable { - - private final Map featureSets; - - public FeatureSetsInfo(StreamInput in) throws IOException { - int size = in.readVInt(); - Map featureSets = new HashMap<>(size); - for (int i = 0; i < size; i++) { - FeatureSet featureSet = new FeatureSet(in); - featureSets.put(featureSet.name, featureSet); - } - this.featureSets = Collections.unmodifiableMap(featureSets); - } - - public FeatureSetsInfo(Set featureSets) { - Map map = new HashMap<>(featureSets.size()); - for (FeatureSet featureSet : featureSets) { - map.put(featureSet.name, featureSet); - } - this.featureSets = Collections.unmodifiableMap(map); - } - - public Map getFeatureSets() { - return featureSets; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - List names = new ArrayList<>(this.featureSets.keySet()).stream().sorted().collect(Collectors.toList()); - for (String name : names) { - builder.field(name, featureSets.get(name), params); - } - return builder.endObject(); - } - - public void writeTo(StreamOutput out) throws IOException { - out.writeVInt(featureSets.size()); - for (FeatureSet featureSet : featureSets.values()) { - featureSet.writeTo(out); - } - } - - public static class FeatureSet implements ToXContentObject, Writeable { - - private final String name; - @Nullable private final String description; - private final boolean available; - private final boolean enabled; - @Nullable private final Map nativeCodeInfo; - - public FeatureSet(StreamInput in) throws IOException { - this(in.readString(), in.readOptionalString(), in.readBoolean(), in.readBoolean(), - in.getVersion().onOrAfter(Version.V_5_4_0) ? in.readMap() : null); - } - - public FeatureSet(String name, @Nullable String description, boolean available, boolean enabled, - @Nullable Map nativeCodeInfo) { - this.name = name; - this.description = description; - this.available = available; - this.enabled = enabled; - this.nativeCodeInfo = nativeCodeInfo; - } - - public String name() { - return name; - } - - @Nullable - public String description() { - return description; - } - - public boolean available() { - return available; - } - - public boolean enabled() { - return enabled; - } - - @Nullable - public Map nativeCodeInfo() { - return nativeCodeInfo; - } - - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - if (description != null) { - builder.field("description", description); - } - builder.field("available", available); - builder.field("enabled", enabled); - if (nativeCodeInfo != null) { - builder.field("native_code_info", nativeCodeInfo); - } - return builder.endObject(); - } - - public void writeTo(StreamOutput out) throws IOException { - out.writeString(name); - out.writeOptionalString(description); - out.writeBoolean(available); - out.writeBoolean(enabled); - if (out.getVersion().onOrAfter(Version.V_5_4_0)) { - out.writeMap(nativeCodeInfo); - } - } - } - - } -} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClient.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClient.java index 588a9c0543379..77f511ba4d0ef 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClient.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClient.java @@ -9,9 +9,9 @@ import org.elasticsearch.client.Client; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.license.LicensingClient; -import org.elasticsearch.license.XPackInfoResponse; +import org.elasticsearch.protocol.xpack.XPackInfoRequest; +import org.elasticsearch.protocol.xpack.XPackInfoResponse; import org.elasticsearch.xpack.core.action.XPackInfoAction; -import org.elasticsearch.xpack.core.action.XPackInfoRequest; import org.elasticsearch.xpack.core.action.XPackInfoRequestBuilder; import org.elasticsearch.xpack.core.ml.client.MachineLearningClient; import org.elasticsearch.xpack.core.monitoring.client.MonitoringClient; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackInfoAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackInfoAction.java index 415e601a40aa9..2fac466b6291c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackInfoAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/TransportXPackInfoAction.java @@ -11,16 +11,18 @@ import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.license.XPackInfoResponse; import org.elasticsearch.license.License; import org.elasticsearch.license.LicenseService; import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.protocol.xpack.XPackInfoRequest; +import org.elasticsearch.protocol.xpack.XPackInfoResponse; +import org.elasticsearch.protocol.xpack.XPackInfoResponse.FeatureSetsInfo.FeatureSet; +import org.elasticsearch.protocol.xpack.XPackInfoResponse.LicenseInfo; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.XPackBuild; import org.elasticsearch.xpack.core.XPackFeatureSet; -import org.elasticsearch.license.XPackInfoResponse.FeatureSetsInfo.FeatureSet; -import org.elasticsearch.license.XPackInfoResponse.LicenseInfo; +import java.util.Locale; import java.util.Set; import java.util.stream.Collectors; @@ -45,14 +47,15 @@ protected void doExecute(XPackInfoRequest request, ActionListener { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoRequestBuilder.java index 6ee4fb925bde2..e7219ed586d07 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoRequestBuilder.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoRequestBuilder.java @@ -7,7 +7,8 @@ import org.elasticsearch.action.ActionRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; -import org.elasticsearch.license.XPackInfoResponse; +import org.elasticsearch.protocol.xpack.XPackInfoRequest; +import org.elasticsearch.protocol.xpack.XPackInfoResponse; import java.util.EnumSet; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rest/action/RestXPackInfoAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rest/action/RestXPackInfoAction.java index e1c694742cba4..c057c04cc637d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rest/action/RestXPackInfoAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/rest/action/RestXPackInfoAction.java @@ -6,15 +6,11 @@ package org.elasticsearch.xpack.core.rest.action; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.license.XPackInfoResponse; -import org.elasticsearch.rest.BytesRestResponse; +import org.elasticsearch.protocol.xpack.XPackInfoRequest; import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.RestResponse; -import org.elasticsearch.rest.action.RestBuilderListener; +import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.xpack.core.XPackClient; -import org.elasticsearch.xpack.core.action.XPackInfoRequest; import org.elasticsearch.xpack.core.rest.XPackRestHandler; import java.io.IOException; @@ -22,7 +18,6 @@ import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestRequest.Method.HEAD; -import static org.elasticsearch.rest.RestStatus.OK; public class RestXPackInfoAction extends XPackRestHandler { public RestXPackInfoAction(Settings settings, RestController controller) { @@ -48,36 +43,6 @@ public RestChannelConsumer doPrepareRequest(RestRequest request, XPackClient cli client.prepareInfo() .setVerbose(verbose) .setCategories(categories) - .execute(new RestBuilderListener(channel) { - @Override - public RestResponse buildResponse(XPackInfoResponse infoResponse, XContentBuilder builder) throws Exception { - - builder.startObject(); - - if (infoResponse.getBuildInfo() != null) { - builder.field("build", infoResponse.getBuildInfo(), request); - } - - if (infoResponse.getLicenseInfo() != null) { - builder.field("license", infoResponse.getLicenseInfo(), request); - } else if (categories.contains(XPackInfoRequest.Category.LICENSE)) { - // if the user requested the license info, and there is no license, we should send - // back an explicit null value (indicating there is no license). This is different - // than not adding the license info at all - builder.nullField("license"); - } - - if (infoResponse.getFeatureSetsInfo() != null) { - builder.field("features", infoResponse.getFeatureSetsInfo(), request); - } - - if (verbose) { - builder.field("tagline", "You know, for X"); - } - - builder.endObject(); - return new BytesRestResponse(OK, builder); - } - }); + .execute(new RestToXContentListener<>(channel)); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/TransportXPackInfoActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/TransportXPackInfoActionTests.java index e46cefb77938e..459f7fab02d72 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/TransportXPackInfoActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/TransportXPackInfoActionTests.java @@ -9,15 +9,17 @@ import org.elasticsearch.action.support.ActionFilters; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.license.XPackInfoResponse; import org.elasticsearch.license.License; import org.elasticsearch.license.LicenseService; +import org.elasticsearch.protocol.license.LicenseStatus; +import org.elasticsearch.protocol.xpack.XPackInfoRequest; +import org.elasticsearch.protocol.xpack.XPackInfoResponse; +import org.elasticsearch.protocol.xpack.XPackInfoResponse.FeatureSetsInfo.FeatureSet; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.Transport; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xpack.core.XPackFeatureSet; -import org.elasticsearch.license.XPackInfoResponse.FeatureSetsInfo.FeatureSet; import java.util.Collections; import java.util.EnumSet; @@ -62,7 +64,7 @@ public void testDoExecute() throws Exception { License license = mock(License.class); long expiryDate = randomLong(); when(license.expiryDate()).thenReturn(expiryDate); - License.Status status = randomFrom(License.Status.values()); + LicenseStatus status = randomFrom(LicenseStatus.values()); when(license.status()).thenReturn(status); String type = randomAlphaOfLength(10); when(license.type()).thenReturn(type); diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/MlRemoteLicenseChecker.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/MlRemoteLicenseChecker.java index b55713f6d0ab7..791be9b7cb430 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/MlRemoteLicenseChecker.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/MlRemoteLicenseChecker.java @@ -13,11 +13,12 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.license.License; -import org.elasticsearch.license.XPackInfoResponse; +import org.elasticsearch.protocol.license.LicenseStatus; +import org.elasticsearch.protocol.xpack.XPackInfoRequest; +import org.elasticsearch.protocol.xpack.XPackInfoResponse; import org.elasticsearch.transport.ActionNotFoundTransportException; import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.xpack.core.action.XPackInfoAction; -import org.elasticsearch.xpack.core.action.XPackInfoRequest; import java.util.EnumSet; import java.util.Iterator; @@ -136,7 +137,7 @@ private void remoteClusterLicense(String clusterName, ActionListener remoteClusterNames(List indices) { public static String buildErrorMessage(RemoteClusterLicenseInfo clusterLicenseInfo) { StringBuilder error = new StringBuilder(); - if (clusterLicenseInfo.licenseInfo.getStatus() != License.Status.ACTIVE) { + if (clusterLicenseInfo.licenseInfo.getStatus() != LicenseStatus.ACTIVE) { error.append("The license on cluster [").append(clusterLicenseInfo.clusterName) .append("] is not active. "); } else { diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/MlRemoteLicenseCheckerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/MlRemoteLicenseCheckerTests.java index dfd7c886ebf42..c10bf1081dd80 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/MlRemoteLicenseCheckerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/MlRemoteLicenseCheckerTests.java @@ -11,8 +11,8 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.license.License; -import org.elasticsearch.license.XPackInfoResponse; +import org.elasticsearch.protocol.license.LicenseStatus; +import org.elasticsearch.protocol.xpack.XPackInfoResponse; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.action.XPackInfoAction; @@ -66,16 +66,16 @@ public void testRemoteClusterNames() { public void testLicenseSupportsML() { XPackInfoResponse.LicenseInfo licenseInfo = new XPackInfoResponse.LicenseInfo("uid", "trial", "trial", - License.Status.ACTIVE, randomNonNegativeLong()); + LicenseStatus.ACTIVE, randomNonNegativeLong()); assertTrue(MlRemoteLicenseChecker.licenseSupportsML(licenseInfo)); - licenseInfo = new XPackInfoResponse.LicenseInfo("uid", "trial", "trial", License.Status.EXPIRED, randomNonNegativeLong()); + licenseInfo = new XPackInfoResponse.LicenseInfo("uid", "trial", "trial", LicenseStatus.EXPIRED, randomNonNegativeLong()); assertFalse(MlRemoteLicenseChecker.licenseSupportsML(licenseInfo)); - licenseInfo = new XPackInfoResponse.LicenseInfo("uid", "GOLD", "GOLD", License.Status.ACTIVE, randomNonNegativeLong()); + licenseInfo = new XPackInfoResponse.LicenseInfo("uid", "GOLD", "GOLD", LicenseStatus.ACTIVE, randomNonNegativeLong()); assertFalse(MlRemoteLicenseChecker.licenseSupportsML(licenseInfo)); - licenseInfo = new XPackInfoResponse.LicenseInfo("uid", "PLATINUM", "PLATINUM", License.Status.ACTIVE, randomNonNegativeLong()); + licenseInfo = new XPackInfoResponse.LicenseInfo("uid", "PLATINUM", "PLATINUM", LicenseStatus.ACTIVE, randomNonNegativeLong()); assertTrue(MlRemoteLicenseChecker.licenseSupportsML(licenseInfo)); } @@ -186,14 +186,14 @@ private Client createMockClient() { } private XPackInfoResponse.LicenseInfo createPlatinumLicenseResponse() { - return new XPackInfoResponse.LicenseInfo("uid", "PLATINUM", "PLATINUM", License.Status.ACTIVE, randomNonNegativeLong()); + return new XPackInfoResponse.LicenseInfo("uid", "PLATINUM", "PLATINUM", LicenseStatus.ACTIVE, randomNonNegativeLong()); } private XPackInfoResponse.LicenseInfo createBasicLicenseResponse() { - return new XPackInfoResponse.LicenseInfo("uid", "BASIC", "BASIC", License.Status.ACTIVE, randomNonNegativeLong()); + return new XPackInfoResponse.LicenseInfo("uid", "BASIC", "BASIC", LicenseStatus.ACTIVE, randomNonNegativeLong()); } private XPackInfoResponse.LicenseInfo createExpiredLicenseResponse() { - return new XPackInfoResponse.LicenseInfo("uid", "PLATINUM", "PLATINUM", License.Status.EXPIRED, randomNonNegativeLong()); + return new XPackInfoResponse.LicenseInfo("uid", "PLATINUM", "PLATINUM", LicenseStatus.EXPIRED, randomNonNegativeLong()); } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/tool/SetupPasswordToolTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/tool/SetupPasswordToolTests.java index 11c6b426c5c18..258909abc267a 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/tool/SetupPasswordToolTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/esnative/tool/SetupPasswordToolTests.java @@ -28,9 +28,9 @@ import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.env.Environment; import org.elasticsearch.env.TestEnvironment; -import org.elasticsearch.license.XPackInfoResponse; -import org.elasticsearch.license.XPackInfoResponse.FeatureSetsInfo; -import org.elasticsearch.license.XPackInfoResponse.FeatureSetsInfo.FeatureSet; +import org.elasticsearch.protocol.xpack.XPackInfoResponse; +import org.elasticsearch.protocol.xpack.XPackInfoResponse.FeatureSetsInfo; +import org.elasticsearch.protocol.xpack.XPackInfoResponse.FeatureSetsInfo.FeatureSet; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.xpack.core.security.support.Validation; import org.elasticsearch.xpack.core.security.user.ElasticUser; diff --git a/x-pack/protocol/LICENSE.txt b/x-pack/protocol/LICENSE.txt new file mode 100644 index 0000000000000..d645695673349 --- /dev/null +++ b/x-pack/protocol/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/x-pack/protocol/build.gradle b/x-pack/protocol/build.gradle new file mode 100644 index 0000000000000..7ca81c05e3947 --- /dev/null +++ b/x-pack/protocol/build.gradle @@ -0,0 +1,29 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +apply plugin: 'elasticsearch.build' + +description = 'Request and Response objects for x-pack that are used by the' + + ' high level rest client and x-pack itself' + +dependencies { + compileOnly "org.elasticsearch:elasticsearch:${version}" + + testCompile "org.elasticsearch.test:framework:${version}" +} diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/license/LicenseStatus.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/license/LicenseStatus.java new file mode 100644 index 0000000000000..f0f49351ab1d8 --- /dev/null +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/license/LicenseStatus.java @@ -0,0 +1,67 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.protocol.license; + +import java.io.IOException; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; + +/** + * Status of an X-Pack license. + */ +public enum LicenseStatus implements Writeable { + + ACTIVE("active"), + INVALID("invalid"), + EXPIRED("expired"); + + private final String label; + + LicenseStatus(String label) { + this.label = label; + } + + public String label() { + return label; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(label); + } + + public static LicenseStatus readFrom(StreamInput in) throws IOException { + return fromString(in.readString()); + } + + public static LicenseStatus fromString(String value) { + switch (value) { + case "active": + return ACTIVE; + case "invalid": + return INVALID; + case "expired": + return EXPIRED; + default: + throw new IllegalArgumentException("unknown license status [" + value + "]"); + } + } +} diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/license/package-info.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/license/package-info.java new file mode 100644 index 0000000000000..f671b280d8495 --- /dev/null +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/license/package-info.java @@ -0,0 +1,24 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * Request and Response objects for the default distribution's License + * APIs. + */ +package org.elasticsearch.protocol.license; diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/security/package-info.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/security/package-info.java new file mode 100644 index 0000000000000..2a0ed5bc52b15 --- /dev/null +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/security/package-info.java @@ -0,0 +1,24 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * Request and Response objects for the default distribution's Security + * APIs. + */ +package org.elasticsearch.protocol.security; diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/watcher/package-info.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/watcher/package-info.java new file mode 100644 index 0000000000000..43d7dd29ec9e5 --- /dev/null +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/watcher/package-info.java @@ -0,0 +1,24 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * Request and Response objects for the default distribution's Watcher + * APIs. + */ +package org.elasticsearch.protocol.watcher; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoRequest.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoRequest.java similarity index 71% rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoRequest.java rename to x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoRequest.java index b3c88be93aaca..ce43b763e2313 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoRequest.java +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoRequest.java @@ -1,9 +1,22 @@ /* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. */ -package org.elasticsearch.xpack.core.action; +package org.elasticsearch.protocol.xpack; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionRequestValidationException; @@ -14,6 +27,9 @@ import java.util.EnumSet; import java.util.Locale; +/** + * Fetch information about X-Pack from the cluster. + */ public class XPackInfoRequest extends ActionRequest { public enum Category { diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoResponse.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoResponse.java new file mode 100644 index 0000000000000..d38174e03f3a4 --- /dev/null +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoResponse.java @@ -0,0 +1,500 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.protocol.xpack; + +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.protocol.license.LicenseStatus; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + +public class XPackInfoResponse extends ActionResponse implements ToXContentObject { + /** + * Value of the license's expiration time if it should never expire. + */ + public static final long BASIC_SELF_GENERATED_LICENSE_EXPIRATION_MILLIS = Long.MAX_VALUE - TimeUnit.HOURS.toMillis(24 * 365); + // TODO move this constant to License.java once we move License.java to the protocol jar + + @Nullable private BuildInfo buildInfo; + @Nullable private LicenseInfo licenseInfo; + @Nullable private FeatureSetsInfo featureSetsInfo; + + public XPackInfoResponse() {} + + public XPackInfoResponse(@Nullable BuildInfo buildInfo, @Nullable LicenseInfo licenseInfo, @Nullable FeatureSetsInfo featureSetsInfo) { + this.buildInfo = buildInfo; + this.licenseInfo = licenseInfo; + this.featureSetsInfo = featureSetsInfo; + } + + /** + * @return The build info (incl. build hash and timestamp) + */ + public BuildInfo getBuildInfo() { + return buildInfo; + } + + /** + * @return The current license info (incl. UID, type/mode. status and expiry date). May return {@code null} when no + * license is currently installed. + */ + public LicenseInfo getLicenseInfo() { + return licenseInfo; + } + + /** + * @return The current status of the feature sets in X-Pack. Feature sets describe the features available/enabled in X-Pack. + */ + public FeatureSetsInfo getFeatureSetsInfo() { + return featureSetsInfo; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeOptionalWriteable(buildInfo); + out.writeOptionalWriteable(licenseInfo); + out.writeOptionalWriteable(featureSetsInfo); + } + + @Override + public void readFrom(StreamInput in) throws IOException { + this.buildInfo = in.readOptionalWriteable(BuildInfo::new); + this.licenseInfo = in.readOptionalWriteable(LicenseInfo::new); + this.featureSetsInfo = in.readOptionalWriteable(FeatureSetsInfo::new); + } + + @Override + public boolean equals(Object other) { + if (other == null || other.getClass() != getClass()) return false; + if (this == other) return true; + XPackInfoResponse rhs = (XPackInfoResponse) other; + return Objects.equals(buildInfo, rhs.buildInfo) + && Objects.equals(licenseInfo, rhs.licenseInfo) + && Objects.equals(featureSetsInfo, rhs.featureSetsInfo); + } + + @Override + public int hashCode() { + return Objects.hash(buildInfo, licenseInfo, featureSetsInfo); + } + + @Override + public String toString() { + return Strings.toString(this, true, false); + } + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "xpack_info_response", true, (a, v) -> { + BuildInfo buildInfo = (BuildInfo) a[0]; + LicenseInfo licenseInfo = (LicenseInfo) a[1]; + @SuppressWarnings("unchecked") // This is how constructing object parser works + List featureSets = (List) a[2]; + FeatureSetsInfo featureSetsInfo = featureSets == null ? null : new FeatureSetsInfo(new HashSet<>(featureSets)); + return new XPackInfoResponse(buildInfo, licenseInfo, featureSetsInfo); + }); + static { + PARSER.declareObject(optionalConstructorArg(), BuildInfo.PARSER, new ParseField("build")); + /* + * licenseInfo is sort of "double optional" because it is + * optional but it can also be send as `null`. + */ + PARSER.declareField(optionalConstructorArg(), (p, v) -> { + if (p.currentToken() == XContentParser.Token.VALUE_NULL) { + return null; + } + return LicenseInfo.PARSER.parse(p, v); + }, + new ParseField("license"), ValueType.OBJECT_OR_NULL); + PARSER.declareNamedObjects(optionalConstructorArg(), + (p, c, name) -> FeatureSetsInfo.FeatureSet.PARSER.parse(p, name), + new ParseField("features")); + } + public static XPackInfoResponse fromXContent(XContentParser parser) throws IOException { + return PARSER.parse(parser, null); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + + if (buildInfo != null) { + builder.field("build", buildInfo, params); + } + + EnumSet categories = XPackInfoRequest.Category + .toSet(Strings.splitStringByCommaToArray(params.param("categories", "_all"))); + if (licenseInfo != null) { + builder.field("license", licenseInfo, params); + } else if (categories.contains(XPackInfoRequest.Category.LICENSE)) { + // if the user requested the license info, and there is no license, we should send + // back an explicit null value (indicating there is no license). This is different + // than not adding the license info at all + builder.nullField("license"); + } + + if (featureSetsInfo != null) { + builder.field("features", featureSetsInfo, params); + } + + if (params.paramAsBoolean("human", true)) { + builder.field("tagline", "You know, for X"); + } + + return builder.endObject(); + } + + public static class LicenseInfo implements ToXContentObject, Writeable { + private final String uid; + private final String type; + private final String mode; + private final LicenseStatus status; + private final long expiryDate; + + public LicenseInfo(String uid, String type, String mode, LicenseStatus status, long expiryDate) { + this.uid = uid; + this.type = type; + this.mode = mode; + this.status = status; + this.expiryDate = expiryDate; + } + + public LicenseInfo(StreamInput in) throws IOException { + this(in.readString(), in.readString(), in.readString(), LicenseStatus.readFrom(in), in.readLong()); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(uid); + out.writeString(type); + out.writeString(mode); + status.writeTo(out); + out.writeLong(expiryDate); + } + + public String getUid() { + return uid; + } + + public String getType() { + return type; + } + + public String getMode() { + return mode; + } + + public long getExpiryDate() { + return expiryDate; + } + + public LicenseStatus getStatus() { + return status; + } + + @Override + public boolean equals(Object other) { + if (other == null || other.getClass() != getClass()) return false; + if (this == other) return true; + LicenseInfo rhs = (LicenseInfo) other; + return Objects.equals(uid, rhs.uid) + && Objects.equals(type, rhs.type) + && Objects.equals(mode, rhs.mode) + && Objects.equals(status, rhs.status) + && expiryDate == rhs.expiryDate; + } + + @Override + public int hashCode() { + return Objects.hash(uid, type, mode, status, expiryDate); + } + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "license_info", true, (a, v) -> { + String uid = (String) a[0]; + String type = (String) a[1]; + String mode = (String) a[2]; + LicenseStatus status = LicenseStatus.fromString((String) a[3]); + Long expiryDate = (Long) a[4]; + long primitiveExpiryDate = expiryDate == null ? BASIC_SELF_GENERATED_LICENSE_EXPIRATION_MILLIS : expiryDate; + return new LicenseInfo(uid, type, mode, status, primitiveExpiryDate); + }); + static { + PARSER.declareString(constructorArg(), new ParseField("uid")); + PARSER.declareString(constructorArg(), new ParseField("type")); + PARSER.declareString(constructorArg(), new ParseField("mode")); + PARSER.declareString(constructorArg(), new ParseField("status")); + PARSER.declareLong(optionalConstructorArg(), new ParseField("expiry_date_in_millis")); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + .field("uid", uid) + .field("type", type) + .field("mode", mode) + .field("status", status.label()); + if (expiryDate != BASIC_SELF_GENERATED_LICENSE_EXPIRATION_MILLIS) { + builder.timeField("expiry_date_in_millis", "expiry_date", expiryDate); + } + return builder.endObject(); + } + } + + public static class BuildInfo implements ToXContentObject, Writeable { + private final String hash; + private final String timestamp; + + public BuildInfo(String hash, String timestamp) { + this.hash = hash; + this.timestamp = timestamp; + } + + public BuildInfo(StreamInput input) throws IOException { + this(input.readString(), input.readString()); + } + + @Override + public void writeTo(StreamOutput output) throws IOException { + output.writeString(hash); + output.writeString(timestamp); + } + + public String getHash() { + return hash; + } + + public String getTimestamp() { + return timestamp; + } + + @Override + public boolean equals(Object other) { + if (other == null || other.getClass() != getClass()) return false; + if (this == other) return true; + BuildInfo rhs = (BuildInfo) other; + return Objects.equals(hash, rhs.hash) + && Objects.equals(timestamp, rhs.timestamp); + } + + @Override + public int hashCode() { + return Objects.hash(hash, timestamp); + } + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "build_info", true, (a, v) -> new BuildInfo((String) a[0], (String) a[1])); + static { + PARSER.declareString(constructorArg(), new ParseField("hash")); + PARSER.declareString(constructorArg(), new ParseField("date")); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.startObject() + .field("hash", hash) + .field("date", timestamp) + .endObject(); + } + } + + public static class FeatureSetsInfo implements ToXContentObject, Writeable { + private final Map featureSets; + + public FeatureSetsInfo(Set featureSets) { + Map map = new HashMap<>(featureSets.size()); + for (FeatureSet featureSet : featureSets) { + map.put(featureSet.name, featureSet); + } + this.featureSets = Collections.unmodifiableMap(map); + } + + public FeatureSetsInfo(StreamInput in) throws IOException { + int size = in.readVInt(); + Map featureSets = new HashMap<>(size); + for (int i = 0; i < size; i++) { + FeatureSet featureSet = new FeatureSet(in); + featureSets.put(featureSet.name, featureSet); + } + this.featureSets = Collections.unmodifiableMap(featureSets); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(featureSets.size()); + for (FeatureSet featureSet : featureSets.values()) { + featureSet.writeTo(out); + } + } + + public Map getFeatureSets() { + return featureSets; + } + + @Override + public boolean equals(Object other) { + if (other == null || other.getClass() != getClass()) return false; + if (this == other) return true; + FeatureSetsInfo rhs = (FeatureSetsInfo) other; + return Objects.equals(featureSets, rhs.featureSets); + } + + @Override + public int hashCode() { + return Objects.hash(featureSets); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + List names = new ArrayList<>(this.featureSets.keySet()).stream().sorted().collect(Collectors.toList()); + for (String name : names) { + builder.field(name, featureSets.get(name), params); + } + return builder.endObject(); + } + + public static class FeatureSet implements ToXContentObject, Writeable { + private final String name; + @Nullable private final String description; + private final boolean available; + private final boolean enabled; + @Nullable private final Map nativeCodeInfo; + + public FeatureSet(String name, @Nullable String description, boolean available, boolean enabled, + @Nullable Map nativeCodeInfo) { + this.name = name; + this.description = description; + this.available = available; + this.enabled = enabled; + this.nativeCodeInfo = nativeCodeInfo; + } + + public FeatureSet(StreamInput in) throws IOException { + this(in.readString(), in.readOptionalString(), in.readBoolean(), in.readBoolean(), + in.getVersion().onOrAfter(Version.V_5_4_0) ? in.readMap() : null); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(name); + out.writeOptionalString(description); + out.writeBoolean(available); + out.writeBoolean(enabled); + if (out.getVersion().onOrAfter(Version.V_5_4_0)) { + out.writeMap(nativeCodeInfo); + } + } + + public String name() { + return name; + } + + @Nullable + public String description() { + return description; + } + + public boolean available() { + return available; + } + + public boolean enabled() { + return enabled; + } + + @Nullable + public Map nativeCodeInfo() { + return nativeCodeInfo; + } + + @Override + public boolean equals(Object other) { + if (other == null || other.getClass() != getClass()) return false; + if (this == other) return true; + FeatureSet rhs = (FeatureSet) other; + return Objects.equals(name, rhs.name) + && Objects.equals(description, rhs.description) + && available == rhs.available + && enabled == rhs.enabled + && Objects.equals(nativeCodeInfo, rhs.nativeCodeInfo); + } + + @Override + public int hashCode() { + return Objects.hash(name, description, available, enabled, nativeCodeInfo); + } + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "feature_set", true, (a, name) -> { + String description = (String) a[0]; + boolean available = (Boolean) a[1]; + boolean enabled = (Boolean) a[2]; + @SuppressWarnings("unchecked") // Matches up with declaration below + Map nativeCodeInfo = (Map) a[3]; + return new FeatureSet(name, description, available, enabled, nativeCodeInfo); + }); + static { + PARSER.declareString(optionalConstructorArg(), new ParseField("description")); + PARSER.declareBoolean(constructorArg(), new ParseField("available")); + PARSER.declareBoolean(constructorArg(), new ParseField("enabled")); + PARSER.declareObject(optionalConstructorArg(), (p, name) -> p.map(), new ParseField("native_code_info")); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + if (description != null) { + builder.field("description", description); + } + builder.field("available", available); + builder.field("enabled", enabled); + if (nativeCodeInfo != null) { + builder.field("native_code_info", nativeCodeInfo); + } + return builder.endObject(); + } + } + } +} diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/package-info.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/package-info.java new file mode 100644 index 0000000000000..fab18ccc637e8 --- /dev/null +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/package-info.java @@ -0,0 +1,23 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * Request and Response objects for miscellaneous X-Pack APIs. + */ +package org.elasticsearch.protocol.xpack; diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/license/LicenseStatusTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/license/LicenseStatusTests.java new file mode 100644 index 0000000000000..2c87645157c78 --- /dev/null +++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/license/LicenseStatusTests.java @@ -0,0 +1,30 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.protocol.license; + +import java.io.IOException; + +import org.elasticsearch.test.ESTestCase; + +public class LicenseStatusTests extends ESTestCase { + public void testSerialization() throws IOException { + LicenseStatus status = randomFrom(LicenseStatus.values()); + assertSame(status, copyWriteable(status, writableRegistry(), LicenseStatus::readFrom)); + } +} diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/XPackInfoResponseTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/XPackInfoResponseTests.java new file mode 100644 index 0000000000000..61a936b0df880 --- /dev/null +++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/XPackInfoResponseTests.java @@ -0,0 +1,160 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.protocol.xpack; + +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.protocol.license.LicenseStatus; +import org.elasticsearch.protocol.xpack.XPackInfoResponse.BuildInfo; +import org.elasticsearch.protocol.xpack.XPackInfoResponse.LicenseInfo; +import org.elasticsearch.protocol.xpack.XPackInfoResponse.FeatureSetsInfo; +import org.elasticsearch.protocol.xpack.XPackInfoResponse.FeatureSetsInfo.FeatureSet; +import org.elasticsearch.test.AbstractStreamableXContentTestCase; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.function.Predicate; +import java.io.IOException; + +public class XPackInfoResponseTests extends AbstractStreamableXContentTestCase { + @Override + protected XPackInfoResponse doParseInstance(XContentParser parser) throws IOException { + return XPackInfoResponse.fromXContent(parser); + } + + @Override + protected XPackInfoResponse createBlankInstance() { + return new XPackInfoResponse(); + } + + @Override + protected Predicate getRandomFieldsExcludeFilter() { + return path -> path.equals("features") + || (path.startsWith("features") && path.endsWith("native_code_info")); + } + + @Override + protected ToXContent.Params getToXContentParams() { + Map params = new HashMap<>(); + if (randomBoolean()) { + params.put("human", randomBoolean() ? "true" : "false"); + } + if (randomBoolean()) { + params.put("categories", "_none"); + } + return new ToXContent.MapParams(params); + } + + @Override + protected XPackInfoResponse createTestInstance() { + return new XPackInfoResponse( + randomBoolean() ? null : randomBuildInfo(), + randomBoolean() ? null : randomLicenseInfo(), + randomBoolean() ? null : randomFeatureSetsInfo()); + } + + @Override + protected XPackInfoResponse mutateInstance(XPackInfoResponse response) { + @SuppressWarnings("unchecked") + Function mutator = randomFrom( + r -> new XPackInfoResponse( + mutateBuildInfo(r.getBuildInfo()), + r.getLicenseInfo(), + r.getFeatureSetsInfo()), + r -> new XPackInfoResponse( + r.getBuildInfo(), + mutateLicenseInfo(r.getLicenseInfo()), + r.getFeatureSetsInfo()), + r -> new XPackInfoResponse( + r.getBuildInfo(), + r.getLicenseInfo(), + mutateFeatureSetsInfo(r.getFeatureSetsInfo()))); + return mutator.apply(response); + } + + private BuildInfo randomBuildInfo() { + return new BuildInfo( + randomAlphaOfLength(10), + randomAlphaOfLength(15)); + } + + private BuildInfo mutateBuildInfo(BuildInfo buildInfo) { + if (buildInfo == null) { + return randomBuildInfo(); + } + return null; + } + + private LicenseInfo randomLicenseInfo() { + return new LicenseInfo( + randomAlphaOfLength(10), + randomAlphaOfLength(4), + randomAlphaOfLength(5), + randomFrom(LicenseStatus.values()), + randomLong()); + } + + private LicenseInfo mutateLicenseInfo(LicenseInfo licenseInfo) { + if (licenseInfo == null) { + return randomLicenseInfo(); + } + return null; + } + + private FeatureSetsInfo randomFeatureSetsInfo() { + int size = between(0, 10); + Set featureSets = new HashSet<>(size); + while (featureSets.size() < size) { + featureSets.add(randomFeatureSet()); + } + return new FeatureSetsInfo(featureSets); + } + + private FeatureSetsInfo mutateFeatureSetsInfo(FeatureSetsInfo featureSetsInfo) { + if (featureSetsInfo == null) { + return randomFeatureSetsInfo(); + } + return null; + } + + private FeatureSet randomFeatureSet() { + return new FeatureSet( + randomAlphaOfLength(5), + randomBoolean() ? null : randomAlphaOfLength(20), + randomBoolean(), + randomBoolean(), + randomNativeCodeInfo()); + } + + private Map randomNativeCodeInfo() { + if (randomBoolean()) { + return null; + } + int size = between(0, 10); + Map nativeCodeInfo = new HashMap<>(size); + while (nativeCodeInfo.size() < size) { + nativeCodeInfo.put(randomAlphaOfLength(5), randomAlphaOfLength(5)); + } + return nativeCodeInfo; + } +} From 942d537d82d8a28c6ec26606cac9677dc4c36aa6 Mon Sep 17 00:00:00 2001 From: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> Date: Fri, 22 Jun 2018 10:15:38 +1000 Subject: [PATCH 04/50] [Security] Check auth scheme case insensitively (#31490) According to RFC 7617, the Basic authentication scheme name should not be case sensitive. Case insensitive comparisons are also applicable for the bearer tokens where Bearer authentication scheme is used as per RFC 6750 and RFC 7235 Some Http clients may send authentication scheme names in different case types for eg. Basic, basic, BASIC, BEARER etc., so the lack of case-insensitive check is an issue when these clients try to authenticate with elasticsearch. This commit adds case-insensitive checks for Basic and Bearer authentication schemes. Closes #31486 --- .../authc/support/UsernamePasswordToken.java | 13 ++++++++----- .../xpack/security/authc/TokenService.java | 2 +- .../security/authc/TokenServiceTests.java | 18 +++++++++++++++++- .../support/UsernamePasswordTokenTests.java | 7 ++++--- 4 files changed, 30 insertions(+), 10 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/UsernamePasswordToken.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/UsernamePasswordToken.java index 4fdf32608dd6a..d8e58c29d237b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/UsernamePasswordToken.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/UsernamePasswordToken.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.core.security.authc.support; +import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.xpack.core.security.authc.AuthenticationToken; @@ -20,6 +21,8 @@ public class UsernamePasswordToken implements AuthenticationToken { public static final String BASIC_AUTH_PREFIX = "Basic "; public static final String BASIC_AUTH_HEADER = "Authorization"; + // authorization scheme check is case-insensitive + private static final boolean IGNORE_CASE_AUTH_HEADER_MATCH = true; private final String username; private final SecureString password; @@ -79,15 +82,15 @@ public int hashCode() { public static UsernamePasswordToken extractToken(ThreadContext context) { String authStr = context.getHeader(BASIC_AUTH_HEADER); - if (authStr == null) { - return null; - } - return extractToken(authStr); } private static UsernamePasswordToken extractToken(String headerValue) { - if (headerValue.startsWith(BASIC_AUTH_PREFIX) == false) { + if (Strings.isNullOrEmpty(headerValue)) { + return null; + } + if (headerValue.regionMatches(IGNORE_CASE_AUTH_HEADER_MATCH, 0, BASIC_AUTH_PREFIX, 0, + BASIC_AUTH_PREFIX.length()) == false) { // the header does not start with 'Basic ' so we cannot use it, but it may be valid for another realm return null; } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java index 4aa4c866c8976..7f0e4a13cccf9 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/TokenService.java @@ -1021,7 +1021,7 @@ private void maybeStartTokenRemover() { */ String getFromHeader(ThreadContext threadContext) { String header = threadContext.getHeader("Authorization"); - if (Strings.hasLength(header) && header.startsWith("Bearer ") + if (Strings.hasText(header) && header.regionMatches(true, 0, "Bearer ", 0, "Bearer ".length()) && header.length() > "Bearer ".length()) { return header.substring("Bearer ".length()); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java index dbce97e31def4..afbd2a6bb6bb1 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/TokenServiceTests.java @@ -81,6 +81,7 @@ import static org.elasticsearch.repositories.ESBlobStoreTestCase.randomBytes; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.eq; @@ -192,7 +193,7 @@ public void testAttachAndGetToken() throws Exception { mockGetTokenFromId(token); ThreadContext requestContext = new ThreadContext(Settings.EMPTY); - requestContext.putHeader("Authorization", "Bearer " + tokenService.getUserTokenString(token)); + requestContext.putHeader("Authorization", randomFrom("Bearer ", "BEARER ", "bearer ") + tokenService.getUserTokenString(token)); try (ThreadContext.StoredContext ignore = requestContext.newStoredContext(true)) { PlainActionFuture future = new PlainActionFuture<>(); @@ -213,6 +214,21 @@ public void testAttachAndGetToken() throws Exception { } } + public void testInvalidAuthorizationHeader() throws Exception { + TokenService tokenService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, securityIndex, clusterService); + ThreadContext requestContext = new ThreadContext(Settings.EMPTY); + String token = randomFrom("", " "); + String authScheme = randomFrom("Bearer ", "BEARER ", "bearer ", "Basic "); + requestContext.putHeader("Authorization", authScheme + token); + + try (ThreadContext.StoredContext ignore = requestContext.newStoredContext(true)) { + PlainActionFuture future = new PlainActionFuture<>(); + tokenService.getAndValidateToken(requestContext, future); + UserToken serialized = future.get(); + assertThat(serialized, nullValue()); + } + } + public void testRotateKey() throws Exception { assumeFalse("internally managed keys do not work in a mixed cluster", mixedCluster); TokenService tokenService = new TokenService(tokenServiceEnabledSettings, systemUTC(), client, securityIndex, clusterService); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/UsernamePasswordTokenTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/UsernamePasswordTokenTests.java index 57c452798844c..86b9635851bb6 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/UsernamePasswordTokenTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/support/UsernamePasswordTokenTests.java @@ -45,7 +45,8 @@ public void testPutToken() throws Exception { public void testExtractToken() throws Exception { ThreadContext threadContext = new ThreadContext(Settings.EMPTY); - String header = "Basic " + Base64.getEncoder().encodeToString("user1:test123".getBytes(StandardCharsets.UTF_8)); + final String header = randomFrom("Basic ", "basic ", "BASIC ") + + Base64.getEncoder().encodeToString("user1:test123".getBytes(StandardCharsets.UTF_8)); threadContext.putHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, header); UsernamePasswordToken token = UsernamePasswordToken.extractToken(threadContext); assertThat(token, notNullValue()); @@ -54,7 +55,7 @@ public void testExtractToken() throws Exception { } public void testExtractTokenInvalid() throws Exception { - String[] invalidValues = { "Basic ", "Basic f" }; + final String[] invalidValues = { "Basic ", "Basic f", "basic " }; for (String value : invalidValues) { ThreadContext threadContext = new ThreadContext(Settings.EMPTY); threadContext.putHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, value); @@ -70,7 +71,7 @@ public void testExtractTokenInvalid() throws Exception { public void testHeaderNotMatchingReturnsNull() { ThreadContext threadContext = new ThreadContext(Settings.EMPTY); - String header = randomFrom("BasicBroken", "invalid", "Basic"); + final String header = randomFrom("Basic", "BasicBroken", "invalid", " basic "); threadContext.putHeader(UsernamePasswordToken.BASIC_AUTH_HEADER, header); UsernamePasswordToken extracted = UsernamePasswordToken.extractToken(threadContext); assertThat(extracted, nullValue()); From 2ba468e763da23b6a71a0f8438d2ed42138a2286 Mon Sep 17 00:00:00 2001 From: Yogesh Gaikwad <902768+bizybot@users.noreply.github.com> Date: Fri, 22 Jun 2018 17:47:20 +1000 Subject: [PATCH 05/50] [PkiRealm] Invalidate cache on role mappings change (#31510) PkiRealm caches successful authentications and provides ways to invalidate the cache. But in some scenario's the cache was not being invalidated on role mapping change. PkiRealm does not inform role mapper to be notified for cache refresh on role mapping updates. The logic in `TransportClearRealmCacheAction#nodeOperation` which gets invoked for refreshing cache on realms, considers null or empty realm names in the request as clear cache on all realms. When LDAP realm is not present then it clears cache for all realms so it works fine, but when LDAP realm is configured then role mapper sends a request with LDAP realm names and so the cache is cleared only for those realms. This commit resolves the issue by registering PkiRealm with role mapper for cache refresh. PkiRealm implements CachingRealm and as it does not extend CachingUsernamePasswordRealm, have modified the interface method `refreshRealmOnChange` to accept CachingRealm. --- .../xpack/core/security/authc/support/CachingRealm.java | 5 +++++ .../org/elasticsearch/xpack/security/authc/pki/PkiRealm.java | 1 + .../xpack/security/authc/support/DnRoleMapper.java | 3 ++- .../xpack/security/authc/support/UserRoleMapper.java | 3 ++- .../security/authc/support/mapper/CompositeRoleMapper.java | 4 ++-- .../authc/support/mapper/NativeRoleMappingStore.java | 4 ++-- .../xpack/security/authc/pki/PkiRealmTests.java | 3 +++ 7 files changed, 17 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/CachingRealm.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/CachingRealm.java index f29e4594b489d..9f561956b9106 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/CachingRealm.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authc/support/CachingRealm.java @@ -13,6 +13,11 @@ */ public interface CachingRealm { + /** + * @return The name of this realm. + */ + String name(); + /** * Expires a single user from the cache identified by the String agument * @param username the identifier of the user to be cleared diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/pki/PkiRealm.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/pki/PkiRealm.java index 0d4596c24b807..968a741fd988c 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/pki/PkiRealm.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/pki/PkiRealm.java @@ -86,6 +86,7 @@ public PkiRealm(RealmConfig config, ResourceWatcherService watcherService, Nativ this.trustManager = trustManagers(config); this.principalPattern = PkiRealmSettings.USERNAME_PATTERN_SETTING.get(config.settings()); this.roleMapper = roleMapper; + this.roleMapper.refreshRealmOnChange(this); this.cache = CacheBuilder.builder() .setExpireAfterWrite(PkiRealmSettings.CACHE_TTL_SETTING.get(config.settings())) .setMaximumWeight(PkiRealmSettings.CACHE_MAX_USERS_SETTING.get(config.settings())) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/DnRoleMapper.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/DnRoleMapper.java index 6516b02f68d0b..20377dc27dd6b 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/DnRoleMapper.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/DnRoleMapper.java @@ -32,6 +32,7 @@ import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.security.authc.RealmConfig; +import org.elasticsearch.xpack.core.security.authc.support.CachingRealm; import org.elasticsearch.xpack.core.security.authc.support.DnRoleMapperSettings; import static java.util.Collections.emptyMap; @@ -69,7 +70,7 @@ public DnRoleMapper(RealmConfig config, ResourceWatcherService watcherService) { } @Override - public void refreshRealmOnChange(CachingUsernamePasswordRealm realm) { + public void refreshRealmOnChange(CachingRealm realm) { addListener(realm::expireAll); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/UserRoleMapper.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/UserRoleMapper.java index ffdab15e3b507..14d51df752abd 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/UserRoleMapper.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/UserRoleMapper.java @@ -15,6 +15,7 @@ import org.elasticsearch.common.Nullable; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.xpack.core.security.authc.RealmConfig; +import org.elasticsearch.xpack.core.security.authc.support.CachingRealm; import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.ExpressionModel; import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.FieldExpression; import org.elasticsearch.xpack.core.security.authz.permission.Role; @@ -44,7 +45,7 @@ public interface UserRoleMapper { * the whole cluster depending on whether this role-mapper has node-local data or cluster-wide * data. */ - void refreshRealmOnChange(CachingUsernamePasswordRealm realm); + void refreshRealmOnChange(CachingRealm realm); /** * A representation of a user for whom roles should be mapped. diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/CompositeRoleMapper.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/CompositeRoleMapper.java index 0814469cfcea7..2ecf7edff8ae8 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/CompositeRoleMapper.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/CompositeRoleMapper.java @@ -16,7 +16,7 @@ import org.elasticsearch.action.support.GroupedActionListener; import org.elasticsearch.watcher.ResourceWatcherService; import org.elasticsearch.xpack.core.security.authc.RealmConfig; -import org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm; +import org.elasticsearch.xpack.core.security.authc.support.CachingRealm; import org.elasticsearch.xpack.security.authc.support.DnRoleMapper; import org.elasticsearch.xpack.security.authc.support.UserRoleMapper; @@ -48,7 +48,7 @@ public void resolveRoles(UserData user, ActionListener> listener) { } @Override - public void refreshRealmOnChange(CachingUsernamePasswordRealm realm) { + public void refreshRealmOnChange(CachingRealm realm) { this.delegates.forEach(mapper -> mapper.refreshRealmOnChange(realm)); } diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java index 66a490ef959a9..37c538fe78c07 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/support/mapper/NativeRoleMappingStore.java @@ -35,7 +35,7 @@ import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping; import org.elasticsearch.xpack.core.security.authc.support.mapper.expressiondsl.ExpressionModel; import org.elasticsearch.xpack.core.security.client.SecurityClient; -import org.elasticsearch.xpack.security.authc.support.CachingUsernamePasswordRealm; +import org.elasticsearch.xpack.core.security.authc.support.CachingRealm; import org.elasticsearch.xpack.security.authc.support.UserRoleMapper; import org.elasticsearch.xpack.security.support.SecurityIndexManager; @@ -374,7 +374,7 @@ public void resolveRoles(UserData user, ActionListener> listener) { * @see ClearRealmCacheAction */ @Override - public void refreshRealmOnChange(CachingUsernamePasswordRealm realm) { + public void refreshRealmOnChange(CachingRealm realm) { realmsToRefresh.add(realm.name()); } } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiRealmTests.java index 74f6598f8dd1c..44d5859d12b67 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/pki/PkiRealmTests.java @@ -50,6 +50,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoMoreInteractions; import static org.mockito.Mockito.when; public class PkiRealmTests extends ESTestCase { @@ -104,6 +105,7 @@ private void assertSuccessfulAuthentication(Set roles) throws Exception UserRoleMapper roleMapper = mock(UserRoleMapper.class); PkiRealm realm = new PkiRealm(new RealmConfig("", Settings.EMPTY, globalSettings, TestEnvironment.newEnvironment(globalSettings), new ThreadContext(globalSettings)), roleMapper); + verify(roleMapper).refreshRealmOnChange(realm); Mockito.doAnswer(invocation -> { final UserRoleMapper.UserData userData = (UserRoleMapper.UserData) invocation.getArguments()[0]; final ActionListener> listener = (ActionListener>) invocation.getArguments()[1]; @@ -144,6 +146,7 @@ private void assertSuccessfulAuthentication(Set roles) throws Exception final int numTimes = invalidate ? 2 : 1; verify(roleMapper, times(numTimes)).resolveRoles(any(UserRoleMapper.UserData.class), any(ActionListener.class)); + verifyNoMoreInteractions(roleMapper); } public void testCustomUsernamePattern() throws Exception { From 3b0f826667a24cc39da3aa2d6363b8cfd88031ed Mon Sep 17 00:00:00 2001 From: Alexander Reelsen Date: Mon, 9 Jul 2018 09:44:06 +0200 Subject: [PATCH 06/50] Watcher: Add ssl.trust email account setting (#31684) In order to allow users to specify hosts, where SSL is always trusted, this setting is exposed. Otherwise the system keystore needs to be configured properly. For more info see https://javaee.github.io/javamail/docs/api/com/sun/mail/smtp/package-summary.html --- .../xpack/watcher/notification/email/EmailService.java | 7 ++++++- .../watcher/notification/email/EmailServiceTests.java | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/EmailService.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/EmailService.java index e0687ee5d6316..3d2ea583eddf2 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/EmailService.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/email/EmailService.java @@ -79,6 +79,10 @@ public class EmailService extends NotificationService { Setting.affixKeySetting("xpack.notification.email.account.", "smtp.local_address", (key) -> Setting.simpleString(key, Property.Dynamic, Property.NodeScope)); + private static final Setting.AffixSetting SETTING_SMTP_SSL_TRUST_ADDRESS = + Setting.affixKeySetting("xpack.notification.email.account.", "smtp.ssl.trust", + (key) -> Setting.simpleString(key, Property.Dynamic, Property.NodeScope)); + private static final Setting.AffixSetting SETTING_SMTP_LOCAL_PORT = Setting.affixKeySetting("xpack.notification.email.account.", "smtp.local_port", (key) -> Setting.intSetting(key, 25, Property.Dynamic, Property.NodeScope)); @@ -110,6 +114,7 @@ public EmailService(Settings settings, @Nullable CryptoService cryptoService, Cl clusterSettings.addAffixUpdateConsumer(SETTING_SMTP_TIMEOUT, (s, o) -> {}, (s, o) -> {}); clusterSettings.addAffixUpdateConsumer(SETTING_SMTP_CONNECTION_TIMEOUT, (s, o) -> {}, (s, o) -> {}); clusterSettings.addAffixUpdateConsumer(SETTING_SMTP_WRITE_TIMEOUT, (s, o) -> {}, (s, o) -> {}); + clusterSettings.addAffixUpdateConsumer(SETTING_SMTP_SSL_TRUST_ADDRESS, (s, o) -> {}, (s, o) -> {}); clusterSettings.addAffixUpdateConsumer(SETTING_SMTP_LOCAL_ADDRESS, (s, o) -> {}, (s, o) -> {}); clusterSettings.addAffixUpdateConsumer(SETTING_SMTP_LOCAL_PORT, (s, o) -> {}, (s, o) -> {}); clusterSettings.addAffixUpdateConsumer(SETTING_SMTP_SEND_PARTIAL, (s, o) -> {}, (s, o) -> {}); @@ -167,7 +172,7 @@ public static List> getSettings() { return Arrays.asList(SETTING_DEFAULT_ACCOUNT, SETTING_PROFILE, SETTING_EMAIL_DEFAULTS, SETTING_SMTP_AUTH, SETTING_SMTP_HOST, SETTING_SMTP_PASSWORD, SETTING_SMTP_PORT, SETTING_SMTP_STARTTLS_ENABLE, SETTING_SMTP_USER, SETTING_SMTP_STARTTLS_REQUIRED, SETTING_SMTP_TIMEOUT, SETTING_SMTP_CONNECTION_TIMEOUT, SETTING_SMTP_WRITE_TIMEOUT, SETTING_SMTP_LOCAL_ADDRESS, - SETTING_SMTP_LOCAL_PORT, SETTING_SMTP_SEND_PARTIAL, SETTING_SMTP_WAIT_ON_QUIT); + SETTING_SMTP_LOCAL_PORT, SETTING_SMTP_SEND_PARTIAL, SETTING_SMTP_WAIT_ON_QUIT, SETTING_SMTP_SSL_TRUST_ADDRESS); } } diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/email/EmailServiceTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/email/EmailServiceTests.java index d2d4e585afbb7..88bc500f10a2d 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/email/EmailServiceTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/email/EmailServiceTests.java @@ -68,6 +68,7 @@ public void testAccountSmtpPropertyConfiguration() { .put("xpack.notification.email.account.account4.smtp.local_port", "1025") .put("xpack.notification.email.account.account5.smtp.host", "localhost") .put("xpack.notification.email.account.account5.smtp.wait_on_quit", true) + .put("xpack.notification.email.account.account5.smtp.ssl.trust", "host1,host2,host3") .build(); EmailService emailService = new EmailService(settings, null, new ClusterSettings(Settings.EMPTY, new HashSet<>(EmailService.getSettings()))); @@ -100,5 +101,6 @@ public void testAccountSmtpPropertyConfiguration() { Account account5 = emailService.getAccount("account5"); Properties properties5 = account5.getConfig().smtp.properties; assertThat(properties5, hasEntry("mail.smtp.quitwait", "true")); + assertThat(properties5, hasEntry("mail.smtp.ssl.trust", "host1,host2,host3")); } } From d061a9a162780df890925cf2e2876cb09e075579 Mon Sep 17 00:00:00 2001 From: Damien Alexandre Date: Mon, 9 Jul 2018 12:48:49 +0200 Subject: [PATCH 07/50] [Docs] Fix typo in the Rollup API Quick Reference (#31855) The Stop endpoint was not correct in the Quick Reference. --- x-pack/docs/en/rollup/api-quickref.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/docs/en/rollup/api-quickref.asciidoc b/x-pack/docs/en/rollup/api-quickref.asciidoc index 10aed1b572d38..937c6a84e5e14 100644 --- a/x-pack/docs/en/rollup/api-quickref.asciidoc +++ b/x-pack/docs/en/rollup/api-quickref.asciidoc @@ -19,7 +19,7 @@ Most {rollup} endpoints have the following base: * {ref}/rollup-get-job.html[GET /job]: List jobs * {ref}/rollup-get-job.html[GET /job/+++]: Get job details * {ref}/rollup-start-job.html[POST /job//_start]: Start a job -* {ref}/rollup-stop-job.html[POST /job/+++]: Stop a job +* {ref}/rollup-stop-job.html[POST /job//_stop]: Stop a job * {ref}/rollup-delete-job.html[DELETE /job/+++]: Delete a job [float] From c9573f65e4919d94fc1b943b38d6421cb808ef63 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Mon, 9 Jul 2018 14:30:01 +0200 Subject: [PATCH 08/50] Ingest: Add ignore_missing option to RemoveProc (#31693) (#31892) Added `ignore_missing` setting to the RemoveProcessor to fix #23086 --- docs/reference/ingest/ingest-node.asciidoc | 5 +++-- .../ingest/common/RemoveProcessor.java | 18 ++++++++++++++--- .../ingest/common/RemoveProcessorTests.java | 20 ++++++++++++++++--- 3 files changed, 35 insertions(+), 8 deletions(-) diff --git a/docs/reference/ingest/ingest-node.asciidoc b/docs/reference/ingest/ingest-node.asciidoc index 0872f00abfee8..aa72c1e868a96 100644 --- a/docs/reference/ingest/ingest-node.asciidoc +++ b/docs/reference/ingest/ingest-node.asciidoc @@ -1767,8 +1767,9 @@ Removes existing fields. If one field doesn't exist, an exception will be thrown .Remove Options [options="header"] |====== -| Name | Required | Default | Description -| `field` | yes | - | Fields to be removed +| Name | Required | Default | Description +| `field` | yes | - | Fields to be removed +| `ignore_missing` | no | `false` | If `true` and `field` does not exist or is `null`, the processor quietly exits without modifying the document |====== Here is an example to remove a single field: diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java index 32d213694b1b5..61dc97ee0ef3c 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RemoveProcessor.java @@ -39,10 +39,12 @@ public final class RemoveProcessor extends AbstractProcessor { public static final String TYPE = "remove"; private final List fields; + private final boolean ignoreMissing; - RemoveProcessor(String tag, List fields) { + RemoveProcessor(String tag, List fields, boolean ignoreMissing) { super(tag); this.fields = new ArrayList<>(fields); + this.ignoreMissing = ignoreMissing; } public List getFields() { @@ -51,7 +53,16 @@ public List getFields() { @Override public void execute(IngestDocument document) { - fields.forEach(document::removeField); + if (ignoreMissing) { + fields.forEach(field -> { + String path = document.renderTemplate(field); + if (document.hasField(path)) { + document.removeField(path); + } + }); + } else { + fields.forEach(document::removeField); + } } @Override @@ -81,7 +92,8 @@ public RemoveProcessor create(Map registry, String pr final List compiledTemplates = fields.stream() .map(f -> ConfigurationUtils.compileTemplate(TYPE, processorTag, "field", f, scriptService)) .collect(Collectors.toList()); - return new RemoveProcessor(processorTag, compiledTemplates); + boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false); + return new RemoveProcessor(processorTag, compiledTemplates, ignoreMissing); } } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java index 641bac8393977..b420b2fe3b00b 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RemoveProcessorTests.java @@ -27,6 +27,7 @@ import java.util.Collections; import java.util.HashMap; +import java.util.Map; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -37,7 +38,7 @@ public void testRemoveFields() throws Exception { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); String field = RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument); Processor processor = new RemoveProcessor(randomAlphaOfLength(10), - Collections.singletonList(new TestTemplateService.MockTemplateScript.Factory(field))); + Collections.singletonList(new TestTemplateService.MockTemplateScript.Factory(field)), false); processor.execute(ingestDocument); assertThat(ingestDocument.hasField(field), equalTo(false)); } @@ -45,8 +46,10 @@ public void testRemoveFields() throws Exception { public void testRemoveNonExistingField() throws Exception { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); String fieldName = RandomDocumentPicks.randomFieldName(random()); - Processor processor = new RemoveProcessor(randomAlphaOfLength(10), - Collections.singletonList(new TestTemplateService.MockTemplateScript.Factory(fieldName))); + Map config = new HashMap<>(); + config.put("field", fieldName); + String processorTag = randomAlphaOfLength(10); + Processor processor = new RemoveProcessor.Factory(TestTemplateService.instance()).create(null, processorTag, config); try { processor.execute(ingestDocument); fail("remove field should have failed"); @@ -54,4 +57,15 @@ public void testRemoveNonExistingField() throws Exception { assertThat(e.getMessage(), containsString("not present as part of path [" + fieldName + "]")); } } + + public void testIgnoreMissing() throws Exception { + IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); + String fieldName = RandomDocumentPicks.randomFieldName(random()); + Map config = new HashMap<>(); + config.put("field", fieldName); + config.put("ignore_missing", true); + String processorTag = randomAlphaOfLength(10); + Processor processor = new RemoveProcessor.Factory(TestTemplateService.instance()).create(null, processorTag, config); + processor.execute(ingestDocument); + } } From 156d5806afeeddf1fea66d49cc81803010b69160 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Mon, 9 Jul 2018 17:36:33 +0200 Subject: [PATCH 09/50] Ingest: Enable Templated Fieldnames in Rename (#31690) (#31896) * Ingest: Enable Templated Fieldnames in Rename --- .../ingest/common/IngestCommonPlugin.java | 2 +- .../ingest/common/RenameProcessor.java | 43 +++++++++++++------ .../common/DotExpanderProcessorTests.java | 4 +- .../common/RenameProcessorFactoryTests.java | 22 ++++++---- .../ingest/common/RenameProcessorTests.java | 29 +++++++------ 5 files changed, 64 insertions(+), 36 deletions(-) diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java index bc900d325104a..d9878cae9e2ab 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java @@ -62,7 +62,7 @@ public Map getProcessors(Processor.Parameters paramet processors.put(DateProcessor.TYPE, new DateProcessor.Factory(parameters.scriptService)); processors.put(SetProcessor.TYPE, new SetProcessor.Factory(parameters.scriptService)); processors.put(AppendProcessor.TYPE, new AppendProcessor.Factory(parameters.scriptService)); - processors.put(RenameProcessor.TYPE, new RenameProcessor.Factory()); + processors.put(RenameProcessor.TYPE, new RenameProcessor.Factory(parameters.scriptService)); processors.put(RemoveProcessor.TYPE, new RemoveProcessor.Factory(parameters.scriptService)); processors.put(SplitProcessor.TYPE, new SplitProcessor.Factory()); processors.put(JoinProcessor.TYPE, new JoinProcessor.Factory()); diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java index 90a6389d82e46..a35a164ddd3f1 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/RenameProcessor.java @@ -23,6 +23,8 @@ import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.Processor; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.TemplateScript; import java.util.Map; @@ -33,22 +35,22 @@ public final class RenameProcessor extends AbstractProcessor { public static final String TYPE = "rename"; - private final String field; - private final String targetField; + private final TemplateScript.Factory field; + private final TemplateScript.Factory targetField; private final boolean ignoreMissing; - RenameProcessor(String tag, String field, String targetField, boolean ignoreMissing) { + RenameProcessor(String tag, TemplateScript.Factory field, TemplateScript.Factory targetField, boolean ignoreMissing) { super(tag); this.field = field; this.targetField = targetField; this.ignoreMissing = ignoreMissing; } - String getField() { + TemplateScript.Factory getField() { return field; } - String getTargetField() { + TemplateScript.Factory getTargetField() { return targetField; } @@ -58,28 +60,30 @@ boolean isIgnoreMissing() { @Override public void execute(IngestDocument document) { - if (document.hasField(field, true) == false) { + String path = document.renderTemplate(field); + if (document.hasField(path, true) == false) { if (ignoreMissing) { return; } else { - throw new IllegalArgumentException("field [" + field + "] doesn't exist"); + throw new IllegalArgumentException("field [" + path + "] doesn't exist"); } } // We fail here if the target field point to an array slot that is out of range. // If we didn't do this then we would fail if we set the value in the target_field // and then on failure processors would not see that value we tried to rename as we already // removed it. - if (document.hasField(targetField, true)) { - throw new IllegalArgumentException("field [" + targetField + "] already exists"); + String target = document.renderTemplate(targetField); + if (document.hasField(target, true)) { + throw new IllegalArgumentException("field [" + target + "] already exists"); } - Object value = document.getFieldValue(field, Object.class); - document.removeField(field); + Object value = document.getFieldValue(path, Object.class); + document.removeField(path); try { - document.setFieldValue(targetField, value); + document.setFieldValue(target, value); } catch (Exception e) { // setting the value back to the original field shouldn't as we just fetched it from that field: - document.setFieldValue(field, value); + document.setFieldValue(path, value); throw e; } } @@ -90,13 +94,24 @@ public String getType() { } public static final class Factory implements Processor.Factory { + + private final ScriptService scriptService; + + public Factory(ScriptService scriptService) { + this.scriptService = scriptService; + } + @Override public RenameProcessor create(Map registry, String processorTag, Map config) throws Exception { String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); + TemplateScript.Factory fieldTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag, + "field", field, scriptService); String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field"); + TemplateScript.Factory targetFieldTemplate = ConfigurationUtils.compileTemplate(TYPE, processorTag, + "target_field", targetField, scriptService); boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false); - return new RenameProcessor(processorTag, field, targetField, ignoreMissing); + return new RenameProcessor(processorTag, fieldTemplate, targetFieldTemplate , ignoreMissing); } } } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java index 199c475a8f0bf..fde7f0c9b8a02 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DotExpanderProcessorTests.java @@ -21,6 +21,7 @@ import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.Processor; +import org.elasticsearch.ingest.TestTemplateService; import org.elasticsearch.test.ESTestCase; import java.util.Collections; @@ -86,7 +87,8 @@ public void testEscapeFields_valueField() throws Exception { // so because foo is no branch field but a value field the `foo.bar` field can't be expanded // into [foo].[bar], so foo should be renamed first into `[foo].[bar]: IngestDocument document = new IngestDocument(source, Collections.emptyMap()); - Processor processor = new RenameProcessor("_tag", "foo", "foo.bar", false); + Processor processor = new RenameProcessor("_tag", new TestTemplateService.MockTemplateScript.Factory("foo"), + new TestTemplateService.MockTemplateScript.Factory("foo.bar"), false); processor.execute(document); processor = new DotExpanderProcessor("_tag", null, "foo.bar"); processor.execute(document); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java index 1eaac36a64cca..9dd799ff33971 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorFactoryTests.java @@ -20,8 +20,11 @@ package org.elasticsearch.ingest.common; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.ingest.TestTemplateService; import org.elasticsearch.test.ESTestCase; +import org.junit.Before; +import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -29,21 +32,26 @@ public class RenameProcessorFactoryTests extends ESTestCase { + private RenameProcessor.Factory factory; + + @Before + public void init() { + factory = new RenameProcessor.Factory(TestTemplateService.instance()); + } + public void testCreate() throws Exception { - RenameProcessor.Factory factory = new RenameProcessor.Factory(); Map config = new HashMap<>(); config.put("field", "old_field"); config.put("target_field", "new_field"); String processorTag = randomAlphaOfLength(10); RenameProcessor renameProcessor = factory.create(null, processorTag, config); assertThat(renameProcessor.getTag(), equalTo(processorTag)); - assertThat(renameProcessor.getField(), equalTo("old_field")); - assertThat(renameProcessor.getTargetField(), equalTo("new_field")); + assertThat(renameProcessor.getField().newInstance(Collections.emptyMap()).execute(), equalTo("old_field")); + assertThat(renameProcessor.getTargetField().newInstance(Collections.emptyMap()).execute(), equalTo("new_field")); assertThat(renameProcessor.isIgnoreMissing(), equalTo(false)); } public void testCreateWithIgnoreMissing() throws Exception { - RenameProcessor.Factory factory = new RenameProcessor.Factory(); Map config = new HashMap<>(); config.put("field", "old_field"); config.put("target_field", "new_field"); @@ -51,13 +59,12 @@ public void testCreateWithIgnoreMissing() throws Exception { String processorTag = randomAlphaOfLength(10); RenameProcessor renameProcessor = factory.create(null, processorTag, config); assertThat(renameProcessor.getTag(), equalTo(processorTag)); - assertThat(renameProcessor.getField(), equalTo("old_field")); - assertThat(renameProcessor.getTargetField(), equalTo("new_field")); + assertThat(renameProcessor.getField().newInstance(Collections.emptyMap()).execute(), equalTo("old_field")); + assertThat(renameProcessor.getTargetField().newInstance(Collections.emptyMap()).execute(), equalTo("new_field")); assertThat(renameProcessor.isIgnoreMissing(), equalTo(true)); } public void testCreateNoFieldPresent() throws Exception { - RenameProcessor.Factory factory = new RenameProcessor.Factory(); Map config = new HashMap<>(); config.put("target_field", "new_field"); try { @@ -69,7 +76,6 @@ public void testCreateNoFieldPresent() throws Exception { } public void testCreateNoToPresent() throws Exception { - RenameProcessor.Factory factory = new RenameProcessor.Factory(); Map config = new HashMap<>(); config.put("field", "old_field"); try { diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java index bf35918ad6e24..5b3d88879ba13 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/RenameProcessorTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.Processor; import org.elasticsearch.ingest.RandomDocumentPicks; +import org.elasticsearch.ingest.TestTemplateService; import org.elasticsearch.test.ESTestCase; import java.util.ArrayList; @@ -45,7 +46,7 @@ public void testRename() throws Exception { do { newFieldName = RandomDocumentPicks.randomFieldName(random()); } while (RandomDocumentPicks.canAddField(newFieldName, ingestDocument) == false || newFieldName.equals(fieldName)); - Processor processor = new RenameProcessor(randomAlphaOfLength(10), fieldName, newFieldName, false); + Processor processor = createRenameProcessor(fieldName, newFieldName, false); processor.execute(ingestDocument); assertThat(ingestDocument.getFieldValue(newFieldName, Object.class), equalTo(fieldValue)); } @@ -63,7 +64,7 @@ public void testRenameArrayElement() throws Exception { document.put("one", one); IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); - Processor processor = new RenameProcessor(randomAlphaOfLength(10), "list.0", "item", false); + Processor processor = createRenameProcessor("list.0", "item", false); processor.execute(ingestDocument); Object actualObject = ingestDocument.getSourceAndMetadata().get("list"); assertThat(actualObject, instanceOf(List.class)); @@ -76,7 +77,7 @@ public void testRenameArrayElement() throws Exception { assertThat(actualObject, instanceOf(String.class)); assertThat(actualObject, equalTo("item1")); - processor = new RenameProcessor(randomAlphaOfLength(10), "list.0", "list.3", false); + processor = createRenameProcessor("list.0", "list.3", false); try { processor.execute(ingestDocument); fail("processor execute should have failed"); @@ -91,7 +92,7 @@ public void testRenameArrayElement() throws Exception { public void testRenameNonExistingField() throws Exception { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); String fieldName = RandomDocumentPicks.randomFieldName(random()); - Processor processor = new RenameProcessor(randomAlphaOfLength(10), fieldName, + Processor processor = createRenameProcessor(fieldName, RandomDocumentPicks.randomFieldName(random()), false); try { processor.execute(ingestDocument); @@ -105,7 +106,7 @@ public void testRenameNonExistingFieldWithIgnoreMissing() throws Exception { IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); String fieldName = RandomDocumentPicks.randomFieldName(random()); - Processor processor = new RenameProcessor(randomAlphaOfLength(10), fieldName, + Processor processor = createRenameProcessor(fieldName, RandomDocumentPicks.randomFieldName(random()), true); processor.execute(ingestDocument); assertIngestDocument(originalIngestDocument, ingestDocument); @@ -114,7 +115,7 @@ public void testRenameNonExistingFieldWithIgnoreMissing() throws Exception { public void testRenameNewFieldAlreadyExists() throws Exception { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random()); String fieldName = RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument); - Processor processor = new RenameProcessor(randomAlphaOfLength(10), RandomDocumentPicks.randomExistingFieldName( + Processor processor = createRenameProcessor(RandomDocumentPicks.randomExistingFieldName( random(), ingestDocument), fieldName, false); try { processor.execute(ingestDocument); @@ -129,7 +130,7 @@ public void testRenameExistingFieldNullValue() throws Exception { String fieldName = RandomDocumentPicks.randomFieldName(random()); ingestDocument.setFieldValue(fieldName, null); String newFieldName = randomValueOtherThanMany(ingestDocument::hasField, () -> RandomDocumentPicks.randomFieldName(random())); - Processor processor = new RenameProcessor(randomAlphaOfLength(10), fieldName, newFieldName, false); + Processor processor = createRenameProcessor(fieldName, newFieldName, false); processor.execute(ingestDocument); assertThat(ingestDocument.hasField(fieldName), equalTo(false)); assertThat(ingestDocument.hasField(newFieldName), equalTo(true)); @@ -149,7 +150,7 @@ public Object put(String key, Object value) { source.put("list", Collections.singletonList("item")); IngestDocument ingestDocument = new IngestDocument(source, Collections.emptyMap()); - Processor processor = new RenameProcessor(randomAlphaOfLength(10), "list", "new_field", false); + Processor processor = createRenameProcessor("list", "new_field", false); try { processor.execute(ingestDocument); fail("processor execute should have failed"); @@ -173,7 +174,7 @@ public Object remove(Object key) { source.put("list", Collections.singletonList("item")); IngestDocument ingestDocument = new IngestDocument(source, Collections.emptyMap()); - Processor processor = new RenameProcessor(randomAlphaOfLength(10), "list", "new_field", false); + Processor processor = createRenameProcessor("list", "new_field", false); try { processor.execute(ingestDocument); fail("processor execute should have failed"); @@ -188,12 +189,12 @@ public void testRenameLeafIntoBranch() throws Exception { Map source = new HashMap<>(); source.put("foo", "bar"); IngestDocument ingestDocument = new IngestDocument(source, Collections.emptyMap()); - Processor processor1 = new RenameProcessor(randomAlphaOfLength(10), "foo", "foo.bar", false); + Processor processor1 = createRenameProcessor("foo", "foo.bar", false); processor1.execute(ingestDocument); assertThat(ingestDocument.getFieldValue("foo", Map.class), equalTo(Collections.singletonMap("bar", "bar"))); assertThat(ingestDocument.getFieldValue("foo.bar", String.class), equalTo("bar")); - Processor processor2 = new RenameProcessor(randomAlphaOfLength(10), "foo.bar", "foo.bar.baz", false); + Processor processor2 = createRenameProcessor( "foo.bar", "foo.bar.baz", false); processor2.execute(ingestDocument); assertThat(ingestDocument.getFieldValue("foo", Map.class), equalTo(Collections.singletonMap("bar", Collections.singletonMap("baz", "bar")))); @@ -201,9 +202,13 @@ public void testRenameLeafIntoBranch() throws Exception { assertThat(ingestDocument.getFieldValue("foo.bar.baz", String.class), equalTo("bar")); // for fun lets try to restore it (which don't allow today) - Processor processor3 = new RenameProcessor(randomAlphaOfLength(10), "foo.bar.baz", "foo", false); + Processor processor3 = createRenameProcessor("foo.bar.baz", "foo", false); Exception e = expectThrows(IllegalArgumentException.class, () -> processor3.execute(ingestDocument)); assertThat(e.getMessage(), equalTo("field [foo] already exists")); } + private RenameProcessor createRenameProcessor(String field, String targetField, boolean ignoreMissing) { + return new RenameProcessor(randomAlphaOfLength(10), new TestTemplateService.MockTemplateScript.Factory(field), + new TestTemplateService.MockTemplateScript.Factory(targetField), ignoreMissing); + } } From cbd309bbb1dd5944e2d38a7eb9ba236676623110 Mon Sep 17 00:00:00 2001 From: Tal Levy Date: Mon, 9 Jul 2018 16:03:17 -0400 Subject: [PATCH 10/50] add support for is_write_index in put-alias body parsing (#31674) * add support for is_write_index in put-alias body parsing The Rest Put-Alias Action does separate parsing of the alias body to construct the IndicesAliasesRequest. This extra parsing was missed in #30703. * test flag was not just ignored by the parser --- .../test/indices.put_alias/10_basic.yml | 24 +++++++++++++++++++ .../indices/RestIndexPutAliasAction.java | 6 +++++ 2 files changed, 30 insertions(+) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_alias/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_alias/10_basic.yml index 32a5be627658b..731917ae1e423 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_alias/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_alias/10_basic.yml @@ -59,3 +59,27 @@ indices.put_alias: index: test_index name: foo + +--- +"Can set is_write_index": + + - skip: + version: " - 6.3.99" + reason: "is_write_index is only available from 6.4.0 on" + + - do: + indices.create: + index: test_index + + - do: + indices.put_alias: + index: test_index + name: test_alias + body: + is_write_index: true + + - do: + indices.get_alias: + index: test_index + name: test_alias + - match: {test_index.aliases.test_alias: { 'is_write_index': true }} diff --git a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndexPutAliasAction.java b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndexPutAliasAction.java index 73e9bad45a57b..6091994db96b7 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndexPutAliasAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/admin/indices/RestIndexPutAliasAction.java @@ -66,6 +66,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC String routing = null; String indexRouting = null; String searchRouting = null; + Boolean writeIndex = null; if (request.hasContent()) { try (XContentParser parser = request.contentParser()) { @@ -90,6 +91,8 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC } else if ("searchRouting".equals(currentFieldName) || "search-routing".equals(currentFieldName) || "search_routing".equals(currentFieldName)) { searchRouting = parser.textOrNull(); + } else if ("is_write_index".equals(currentFieldName)) { + writeIndex = parser.booleanValue(); } } else if (token == XContentParser.Token.START_OBJECT) { if ("filter".equals(currentFieldName)) { @@ -117,6 +120,9 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC if (filter != null) { aliasAction.filter(filter); } + if (writeIndex != null) { + aliasAction.writeIndex(writeIndex); + } indicesAliasesRequest.addAliasAction(aliasAction); return channel -> client.admin().indices().aliases(indicesAliasesRequest, new RestToXContentListener<>(channel)); } From d8134402bd2c925d4156b9fca47ae4c1d4d34a12 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 9 Jul 2018 13:40:23 -0700 Subject: [PATCH 11/50] Add opaque_id to audit logging (#31878) Logs opaque_id if it is available with all audit log messages. Relates #31521 --- .../audit/logfile/LoggingAuditTrail.java | 183 ++++++++++-------- .../audit/logfile/LoggingAuditTrailTests.java | 91 +++++---- 2 files changed, 151 insertions(+), 123 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java index 3b9a42179a577..54e10925a985a 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.node.Node; import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.tasks.Task; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportMessage; import org.elasticsearch.xpack.core.security.authc.Authentication; @@ -181,11 +182,11 @@ public void authenticationSuccess(String realm, User user, RestRequest request) if (events.contains(AUTHENTICATION_SUCCESS) && (eventFilterPolicyRegistry.ignorePredicate() .test(new AuditEventMetaInfo(Optional.of(user), Optional.of(realm), Optional.empty(), Optional.empty())) == false)) { if (includeRequestBody) { - logger.info("{}[rest] [authentication_success]\t{}, realm=[{}], uri=[{}], params=[{}], request_body=[{}]", - localNodeInfo.prefix, principal(user), realm, request.uri(), request.params(), restRequestContent(request)); + logger.info("{}[rest] [authentication_success]\t{}, realm=[{}], uri=[{}], params=[{}]{}, request_body=[{}]", + localNodeInfo.prefix, principal(user), realm, request.uri(), request.params(), opaqueId(), restRequestContent(request)); } else { - logger.info("{}[rest] [authentication_success]\t{}, realm=[{}], uri=[{}], params=[{}]", localNodeInfo.prefix, - principal(user), realm, request.uri(), request.params()); + logger.info("{}[rest] [authentication_success]\t{}, realm=[{}], uri=[{}], params=[{}]{}", + localNodeInfo.prefix, principal(user), realm, request.uri(), request.params(), opaqueId()); } } } @@ -198,13 +199,13 @@ public void authenticationSuccess(String realm, User user, String action, Transp .test(new AuditEventMetaInfo(Optional.of(user), Optional.of(realm), Optional.empty(), indices)) == false) { final LocalNodeInfo localNodeInfo = this.localNodeInfo; if (indices.isPresent()) { - logger.info("{}[transport] [authentication_success]\t{}, {}, realm=[{}], action=[{}], indices=[{}], request=[{}]", + logger.info("{}[transport] [authentication_success]\t{}, {}, realm=[{}], action=[{}], indices=[{}], request=[{}]{}", localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), principal(user), realm, action, - arrayToCommaDelimitedString(indices.get()), message.getClass().getSimpleName()); + arrayToCommaDelimitedString(indices.get()), message.getClass().getSimpleName(), opaqueId()); } else { - logger.info("{}[transport] [authentication_success]\t{}, {}, realm=[{}], action=[{}], request=[{}]", + logger.info("{}[transport] [authentication_success]\t{}, {}, realm=[{}], action=[{}], request=[{}]{}", localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), principal(user), realm, action, - message.getClass().getSimpleName()); + message.getClass().getSimpleName(), opaqueId()); } } } @@ -218,12 +219,13 @@ public void anonymousAccessDenied(String action, TransportMessage message) { .test(new AuditEventMetaInfo(Optional.empty(), Optional.empty(), indices)) == false) { final LocalNodeInfo localNodeInfo = this.localNodeInfo; if (indices.isPresent()) { - logger.info("{}[transport] [anonymous_access_denied]\t{}, action=[{}], indices=[{}], request=[{}]", + logger.info("{}[transport] [anonymous_access_denied]\t{}, action=[{}], indices=[{}], request=[{}]{}", localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), action, - arrayToCommaDelimitedString(indices.get()), message.getClass().getSimpleName()); + arrayToCommaDelimitedString(indices.get()), message.getClass().getSimpleName(), opaqueId()); } else { - logger.info("{}[transport] [anonymous_access_denied]\t{}, action=[{}], request=[{}]", localNodeInfo.prefix, - originAttributes(threadContext, message, localNodeInfo), action, message.getClass().getSimpleName()); + logger.info("{}[transport] [anonymous_access_denied]\t{}, action=[{}], request=[{}]{}", + localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), action, + message.getClass().getSimpleName(), opaqueId()); } } } @@ -234,11 +236,11 @@ public void anonymousAccessDenied(RestRequest request) { if (events.contains(ANONYMOUS_ACCESS_DENIED) && (eventFilterPolicyRegistry.ignorePredicate().test(AuditEventMetaInfo.EMPTY) == false)) { if (includeRequestBody) { - logger.info("{}[rest] [anonymous_access_denied]\t{}, uri=[{}], request_body=[{}]", localNodeInfo.prefix, - hostAttributes(request), request.uri(), restRequestContent(request)); + logger.info("{}[rest] [anonymous_access_denied]\t{}, uri=[{}]{}, request_body=[{}]", localNodeInfo.prefix, + hostAttributes(request), request.uri(), opaqueId(), restRequestContent(request)); } else { - logger.info("{}[rest] [anonymous_access_denied]\t{}, uri=[{}]", localNodeInfo.prefix, hostAttributes(request), - request.uri()); + logger.info("{}[rest] [anonymous_access_denied]\t{}, uri=[{}]{}", localNodeInfo.prefix, + hostAttributes(request), request.uri(), opaqueId()); } } } @@ -251,13 +253,13 @@ public void authenticationFailed(AuthenticationToken token, String action, Trans .test(new AuditEventMetaInfo(Optional.of(token), Optional.empty(), indices)) == false) { final LocalNodeInfo localNodeInfo = this.localNodeInfo; if (indices.isPresent()) { - logger.info("{}[transport] [authentication_failed]\t{}, principal=[{}], action=[{}], indices=[{}], request=[{}]", + logger.info("{}[transport] [authentication_failed]\t{}, principal=[{}], action=[{}], indices=[{}], request=[{}]{}", localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), token.principal(), action, - arrayToCommaDelimitedString(indices.get()), message.getClass().getSimpleName()); + arrayToCommaDelimitedString(indices.get()), message.getClass().getSimpleName(), opaqueId()); } else { - logger.info("{}[transport] [authentication_failed]\t{}, principal=[{}], action=[{}], request=[{}]", + logger.info("{}[transport] [authentication_failed]\t{}, principal=[{}], action=[{}], request=[{}]{}", localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), token.principal(), action, - message.getClass().getSimpleName()); + message.getClass().getSimpleName(), opaqueId()); } } } @@ -268,10 +270,11 @@ public void authenticationFailed(RestRequest request) { if (events.contains(AUTHENTICATION_FAILED) && (eventFilterPolicyRegistry.ignorePredicate().test(AuditEventMetaInfo.EMPTY) == false)) { if (includeRequestBody) { - logger.info("{}[rest] [authentication_failed]\t{}, uri=[{}], request_body=[{}]", localNodeInfo.prefix, - hostAttributes(request), request.uri(), restRequestContent(request)); + logger.info("{}[rest] [authentication_failed]\t{}, uri=[{}]{}, request_body=[{}]", localNodeInfo.prefix, + hostAttributes(request), request.uri(), opaqueId(), restRequestContent(request)); } else { - logger.info("{}[rest] [authentication_failed]\t{}, uri=[{}]", localNodeInfo.prefix, hostAttributes(request), request.uri()); + logger.info("{}[rest] [authentication_failed]\t{}, uri=[{}]{}", localNodeInfo.prefix, + hostAttributes(request), request.uri(), opaqueId()); } } } @@ -284,12 +287,13 @@ public void authenticationFailed(String action, TransportMessage message) { .test(new AuditEventMetaInfo(Optional.empty(), Optional.empty(), indices)) == false) { final LocalNodeInfo localNodeInfo = this.localNodeInfo; if (indices.isPresent()) { - logger.info("{}[transport] [authentication_failed]\t{}, action=[{}], indices=[{}], request=[{}]", localNodeInfo.prefix, - originAttributes(threadContext, message, localNodeInfo), action, arrayToCommaDelimitedString(indices.get()), - message.getClass().getSimpleName()); + logger.info("{}[transport] [authentication_failed]\t{}, action=[{}], indices=[{}], request=[{}]{}", + localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), action, + arrayToCommaDelimitedString(indices.get()), message.getClass().getSimpleName(), opaqueId()); } else { - logger.info("{}[transport] [authentication_failed]\t{}, action=[{}], request=[{}]", localNodeInfo.prefix, - originAttributes(threadContext, message, localNodeInfo), action, message.getClass().getSimpleName()); + logger.info("{}[transport] [authentication_failed]\t{}, action=[{}], request=[{}]{}", + localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), action, + message.getClass().getSimpleName(), opaqueId()); } } } @@ -301,11 +305,12 @@ public void authenticationFailed(AuthenticationToken token, RestRequest request) && (eventFilterPolicyRegistry.ignorePredicate() .test(new AuditEventMetaInfo(Optional.of(token), Optional.empty(), Optional.empty())) == false)) { if (includeRequestBody) { - logger.info("{}[rest] [authentication_failed]\t{}, principal=[{}], uri=[{}], request_body=[{}]", localNodeInfo.prefix, - hostAttributes(request), token.principal(), request.uri(), restRequestContent(request)); + logger.info("{}[rest] [authentication_failed]\t{}, principal=[{}], uri=[{}]{}, request_body=[{}]", + localNodeInfo.prefix, hostAttributes(request), token.principal(), request.uri(), opaqueId(), + restRequestContent(request)); } else { - logger.info("{}[rest] [authentication_failed]\t{}, principal=[{}], uri=[{}]", localNodeInfo.prefix, hostAttributes(request), - token.principal(), request.uri()); + logger.info("{}[rest] [authentication_failed]\t{}, principal=[{}], uri=[{}]{}", + localNodeInfo.prefix, hostAttributes(request), token.principal(), request.uri(), opaqueId()); } } } @@ -320,13 +325,13 @@ public void authenticationFailed(String realm, AuthenticationToken token, String if (indices.isPresent()) { logger.info( "{}[transport] [realm_authentication_failed]\trealm=[{}], {}, principal=[{}], action=[{}], indices=[{}], " - + "request=[{}]", + + "request=[{}]{}", localNodeInfo.prefix, realm, originAttributes(threadContext, message, localNodeInfo), token.principal(), action, - arrayToCommaDelimitedString(indices.get()), message.getClass().getSimpleName()); + arrayToCommaDelimitedString(indices.get()), message.getClass().getSimpleName(), opaqueId()); } else { - logger.info("{}[transport] [realm_authentication_failed]\trealm=[{}], {}, principal=[{}], action=[{}], request=[{}]", + logger.info("{}[transport] [realm_authentication_failed]\trealm=[{}], {}, principal=[{}], action=[{}], request=[{}]{}", localNodeInfo.prefix, realm, originAttributes(threadContext, message, localNodeInfo), token.principal(), action, - message.getClass().getSimpleName()); + message.getClass().getSimpleName(), opaqueId()); } } } @@ -338,12 +343,13 @@ public void authenticationFailed(String realm, AuthenticationToken token, RestRe && (eventFilterPolicyRegistry.ignorePredicate() .test(new AuditEventMetaInfo(Optional.of(token), Optional.of(realm), Optional.empty())) == false)) { if (includeRequestBody) { - logger.info("{}[rest] [realm_authentication_failed]\trealm=[{}], {}, principal=[{}], uri=[{}], request_body=[{}]", - localNodeInfo.prefix, realm, hostAttributes(request), token.principal(), request.uri(), + logger.info("{}[rest] [realm_authentication_failed]\trealm=[{}], {}, principal=[{}], uri=[{}]{}, " + + "request_body=[{}]", + localNodeInfo.prefix, realm, hostAttributes(request), token.principal(), request.uri(), opaqueId(), restRequestContent(request)); } else { - logger.info("{}[rest] [realm_authentication_failed]\trealm=[{}], {}, principal=[{}], uri=[{}]", localNodeInfo.prefix, realm, - hostAttributes(request), token.principal(), request.uri()); + logger.info("{}[rest] [realm_authentication_failed]\trealm=[{}], {}, principal=[{}], uri=[{}]{}", + localNodeInfo.prefix, realm, hostAttributes(request), token.principal(), request.uri(), opaqueId()); } } } @@ -358,14 +364,14 @@ public void accessGranted(Authentication authentication, String action, Transpor Optional.of(effectiveRealmName(authentication)), Optional.of(roleNames), indices)) == false) { final LocalNodeInfo localNodeInfo = this.localNodeInfo; if (indices.isPresent()) { - logger.info("{}[transport] [access_granted]\t{}, {}, roles=[{}], action=[{}], indices=[{}], request=[{}]", + logger.info("{}[transport] [access_granted]\t{}, {}, roles=[{}], action=[{}], indices=[{}], request=[{}]{}", localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), subject(authentication), arrayToCommaDelimitedString(roleNames), action, arrayToCommaDelimitedString(indices.get()), - message.getClass().getSimpleName()); + message.getClass().getSimpleName(), opaqueId()); } else { - logger.info("{}[transport] [access_granted]\t{}, {}, roles=[{}], action=[{}], request=[{}]", localNodeInfo.prefix, - originAttributes(threadContext, message, localNodeInfo), subject(authentication), - arrayToCommaDelimitedString(roleNames), action, message.getClass().getSimpleName()); + logger.info("{}[transport] [access_granted]\t{}, {}, roles=[{}], action=[{}], request=[{}]{}", + localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), subject(authentication), + arrayToCommaDelimitedString(roleNames), action, message.getClass().getSimpleName(), opaqueId()); } } } @@ -379,14 +385,14 @@ public void accessDenied(Authentication authentication, String action, Transport Optional.of(effectiveRealmName(authentication)), Optional.of(roleNames), indices)) == false) { final LocalNodeInfo localNodeInfo = this.localNodeInfo; if (indices.isPresent()) { - logger.info("{}[transport] [access_denied]\t{}, {}, roles=[{}], action=[{}], indices=[{}], request=[{}]", + logger.info("{}[transport] [access_denied]\t{}, {}, roles=[{}], action=[{}], indices=[{}], request=[{}]{}", localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), subject(authentication), arrayToCommaDelimitedString(roleNames), action, arrayToCommaDelimitedString(indices.get()), - message.getClass().getSimpleName()); + message.getClass().getSimpleName(), opaqueId()); } else { - logger.info("{}[transport] [access_denied]\t{}, {}, roles=[{}], action=[{}], request=[{}]", localNodeInfo.prefix, - originAttributes(threadContext, message, localNodeInfo), subject(authentication), - arrayToCommaDelimitedString(roleNames), action, message.getClass().getSimpleName()); + logger.info("{}[transport] [access_denied]\t{}, {}, roles=[{}], action=[{}], request=[{}]{}", + localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), subject(authentication), + arrayToCommaDelimitedString(roleNames), action, message.getClass().getSimpleName(), opaqueId()); } } } @@ -396,10 +402,11 @@ localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), s public void tamperedRequest(RestRequest request) { if (events.contains(TAMPERED_REQUEST) && (eventFilterPolicyRegistry.ignorePredicate().test(AuditEventMetaInfo.EMPTY) == false)) { if (includeRequestBody) { - logger.info("{}[rest] [tampered_request]\t{}, uri=[{}], request_body=[{}]", localNodeInfo.prefix, hostAttributes(request), - request.uri(), restRequestContent(request)); + logger.info("{}[rest] [tampered_request]\t{}, uri=[{}]{}, request_body=[{}]", localNodeInfo.prefix, + hostAttributes(request), request.uri(), opaqueId(), restRequestContent(request)); } else { - logger.info("{}[rest] [tampered_request]\t{}, uri=[{}]", localNodeInfo.prefix, hostAttributes(request), request.uri()); + logger.info("{}[rest] [tampered_request]\t{}, uri=[{}]{}", localNodeInfo.prefix, hostAttributes(request), + request.uri(), opaqueId()); } } } @@ -412,12 +419,13 @@ public void tamperedRequest(String action, TransportMessage message) { .test(new AuditEventMetaInfo(Optional.empty(), Optional.empty(), indices)) == false) { final LocalNodeInfo localNodeInfo = this.localNodeInfo; if (indices.isPresent()) { - logger.info("{}[transport] [tampered_request]\t{}, action=[{}], indices=[{}], request=[{}]", localNodeInfo.prefix, - originAttributes(threadContext, message, localNodeInfo), action, arrayToCommaDelimitedString(indices.get()), - message.getClass().getSimpleName()); + logger.info("{}[transport] [tampered_request]\t{}, action=[{}], indices=[{}], request=[{}]{}", + localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), action, + arrayToCommaDelimitedString(indices.get()), message.getClass().getSimpleName(), opaqueId()); } else { - logger.info("{}[transport] [tampered_request]\t{}, action=[{}], request=[{}]", localNodeInfo.prefix, - originAttributes(threadContext, message, localNodeInfo), action, message.getClass().getSimpleName()); + logger.info("{}[transport] [tampered_request]\t{}, action=[{}], request=[{}]{}", localNodeInfo.prefix, + originAttributes(threadContext, message, localNodeInfo), action, message.getClass().getSimpleName(), + opaqueId()); } } } @@ -431,13 +439,13 @@ public void tamperedRequest(User user, String action, TransportMessage request) .test(new AuditEventMetaInfo(Optional.of(user), Optional.empty(), Optional.empty(), indices)) == false) { final LocalNodeInfo localNodeInfo = this.localNodeInfo; if (indices.isPresent()) { - logger.info("{}[transport] [tampered_request]\t{}, {}, action=[{}], indices=[{}], request=[{}]", localNodeInfo.prefix, - originAttributes(threadContext, request, localNodeInfo), principal(user), action, - arrayToCommaDelimitedString(indices.get()), request.getClass().getSimpleName()); + logger.info("{}[transport] [tampered_request]\t{}, {}, action=[{}], indices=[{}], request=[{}]{}", + localNodeInfo.prefix, originAttributes(threadContext, request, localNodeInfo), principal(user), action, + arrayToCommaDelimitedString(indices.get()), request.getClass().getSimpleName(), opaqueId()); } else { - logger.info("{}[transport] [tampered_request]\t{}, {}, action=[{}], request=[{}]", localNodeInfo.prefix, + logger.info("{}[transport] [tampered_request]\t{}, {}, action=[{}], request=[{}]{}", localNodeInfo.prefix, originAttributes(threadContext, request, localNodeInfo), principal(user), action, - request.getClass().getSimpleName()); + request.getClass().getSimpleName(), opaqueId()); } } } @@ -446,16 +454,16 @@ public void tamperedRequest(User user, String action, TransportMessage request) @Override public void connectionGranted(InetAddress inetAddress, String profile, SecurityIpFilterRule rule) { if (events.contains(CONNECTION_GRANTED) && (eventFilterPolicyRegistry.ignorePredicate().test(AuditEventMetaInfo.EMPTY) == false)) { - logger.info("{}[ip_filter] [connection_granted]\torigin_address=[{}], transport_profile=[{}], rule=[{}]", localNodeInfo.prefix, - NetworkAddress.format(inetAddress), profile, rule); + logger.info("{}[ip_filter] [connection_granted]\torigin_address=[{}], transport_profile=[{}], rule=[{}]{}", + localNodeInfo.prefix, NetworkAddress.format(inetAddress), profile, rule, opaqueId()); } } @Override public void connectionDenied(InetAddress inetAddress, String profile, SecurityIpFilterRule rule) { if (events.contains(CONNECTION_DENIED) && (eventFilterPolicyRegistry.ignorePredicate().test(AuditEventMetaInfo.EMPTY) == false)) { - logger.info("{}[ip_filter] [connection_denied]\torigin_address=[{}], transport_profile=[{}], rule=[{}]", localNodeInfo.prefix, - NetworkAddress.format(inetAddress), profile, rule); + logger.info("{}[ip_filter] [connection_denied]\torigin_address=[{}], transport_profile=[{}], rule=[{}]{}", + localNodeInfo.prefix, NetworkAddress.format(inetAddress), profile, rule, opaqueId()); } } @@ -467,15 +475,14 @@ public void runAsGranted(Authentication authentication, String action, Transport Optional.of(effectiveRealmName(authentication)), Optional.of(roleNames), indices)) == false) { final LocalNodeInfo localNodeInfo = this.localNodeInfo; if (indices.isPresent()) { - logger.info("{}[transport] [run_as_granted]\t{}, {}, roles=[{}], action=[{}], indices=[{}], request=[{}]", + logger.info("{}[transport] [run_as_granted]\t{}, {}, roles=[{}], action=[{}], indices=[{}], request=[{}]{}", localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), runAsSubject(authentication), arrayToCommaDelimitedString(roleNames), action, arrayToCommaDelimitedString(indices.get()), - message.getClass().getSimpleName()); + message.getClass().getSimpleName(), opaqueId()); } else { - logger.info("{}[transport] [run_as_granted]\t{}, {}, roles=[{}], action=[{}], request=[{}]", localNodeInfo.prefix, - originAttributes(threadContext, message, localNodeInfo), runAsSubject(authentication), - arrayToCommaDelimitedString(roleNames), action, - message.getClass().getSimpleName()); + logger.info("{}[transport] [run_as_granted]\t{}, {}, roles=[{}], action=[{}], request=[{}]{}", + localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), runAsSubject(authentication), + arrayToCommaDelimitedString(roleNames), action, message.getClass().getSimpleName(), opaqueId()); } } } @@ -489,14 +496,14 @@ public void runAsDenied(Authentication authentication, String action, TransportM Optional.of(effectiveRealmName(authentication)), Optional.of(roleNames), indices)) == false) { final LocalNodeInfo localNodeInfo = this.localNodeInfo; if (indices.isPresent()) { - logger.info("{}[transport] [run_as_denied]\t{}, {}, roles=[{}], action=[{}], indices=[{}], request=[{}]", + logger.info("{}[transport] [run_as_denied]\t{}, {}, roles=[{}], action=[{}], indices=[{}], request=[{}]{}", localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), runAsSubject(authentication), arrayToCommaDelimitedString(roleNames), action, arrayToCommaDelimitedString(indices.get()), - message.getClass().getSimpleName()); + message.getClass().getSimpleName(), opaqueId()); } else { - logger.info("{}[transport] [run_as_denied]\t{}, {}, roles=[{}], action=[{}], request=[{}]", localNodeInfo.prefix, - originAttributes(threadContext, message, localNodeInfo), runAsSubject(authentication), - arrayToCommaDelimitedString(roleNames), action, message.getClass().getSimpleName()); + logger.info("{}[transport] [run_as_denied]\t{}, {}, roles=[{}], action=[{}], request=[{}]{}", + localNodeInfo.prefix, originAttributes(threadContext, message, localNodeInfo), runAsSubject(authentication), + arrayToCommaDelimitedString(roleNames), action, message.getClass().getSimpleName(), opaqueId()); } } } @@ -508,12 +515,13 @@ public void runAsDenied(Authentication authentication, RestRequest request, Stri && (eventFilterPolicyRegistry.ignorePredicate().test(new AuditEventMetaInfo(Optional.of(authentication.getUser()), Optional.of(effectiveRealmName(authentication)), Optional.of(roleNames), Optional.empty())) == false)) { if (includeRequestBody) { - logger.info("{}[rest] [run_as_denied]\t{}, {}, roles=[{}], uri=[{}], request_body=[{}]", localNodeInfo.prefix, - hostAttributes(request), runAsSubject(authentication), arrayToCommaDelimitedString(roleNames), request.uri(), - restRequestContent(request)); + logger.info("{}[rest] [run_as_denied]\t{}, {}, roles=[{}], uri=[{}], request_body=[{}]{}", + localNodeInfo.prefix, hostAttributes(request), runAsSubject(authentication), + arrayToCommaDelimitedString(roleNames), request.uri(), restRequestContent(request), opaqueId()); } else { - logger.info("{}[rest] [run_as_denied]\t{}, {}, roles=[{}], uri=[{}]", localNodeInfo.prefix, hostAttributes(request), - runAsSubject(authentication), arrayToCommaDelimitedString(roleNames), request.uri()); + logger.info("{}[rest] [run_as_denied]\t{}, {}, roles=[{}], uri=[{}]{}", localNodeInfo.prefix, + hostAttributes(request), runAsSubject(authentication), arrayToCommaDelimitedString(roleNames), request.uri(), + opaqueId()); } } } @@ -558,6 +566,15 @@ protected static String originAttributes(ThreadContext threadContext, TransportM return restOriginTag(threadContext).orElse(transportOriginTag(message).orElse(localNodeInfo.localOriginTag)); } + private String opaqueId() { + String opaqueId = threadContext.getHeader(Task.X_OPAQUE_ID); + if (opaqueId != null) { + return ", opaque_id=[" + opaqueId + "]"; + } else { + return ""; + } + } + private static Optional restOriginTag(ThreadContext threadContext) { final InetSocketAddress restAddress = RemoteHostHeader.restRemoteAddress(threadContext); if (restAddress == null) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java index 408d3e797e5a3..1059e22abd663 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrailTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.mock.orig.Mockito; import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.tasks.Task; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.rest.FakeRestRequest; import org.elasticsearch.test.rest.FakeRestRequest.Builder; @@ -122,6 +123,7 @@ protected String expectedMessage() { private ClusterService clusterService; private ThreadContext threadContext; private boolean includeRequestBody; + private String opaqueId; @Before public void init() throws Exception { @@ -145,6 +147,13 @@ public void init() throws Exception { when(clusterService.getClusterSettings()).thenReturn(clusterSettings); prefix = LoggingAuditTrail.LocalNodeInfo.resolvePrefix(settings, localNode); threadContext = new ThreadContext(Settings.EMPTY); + if (randomBoolean()) { + String id = randomAlphaOfLength(10); + threadContext.putHeader(Task.X_OPAQUE_ID, id); + opaqueId = ", opaque_id=[" + id + "]"; + } else { + opaqueId = ""; + } } public void testAnonymousAccessDeniedTransport() throws Exception { @@ -155,10 +164,10 @@ public void testAnonymousAccessDeniedTransport() throws Exception { auditTrail.anonymousAccessDenied("_action", message); if (message instanceof IndicesRequest) { assertMsg(logger, Level.INFO, prefix + "[transport] [anonymous_access_denied]\t" + origins + - ", action=[_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]"); + ", action=[_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]" + opaqueId); } else { assertMsg(logger, Level.INFO, prefix + "[transport] [anonymous_access_denied]\t" + origins + - ", action=[_action], request=[MockMessage]"); + ", action=[_action], request=[MockMessage]" + opaqueId); } // test disabled @@ -179,10 +188,10 @@ public void testAnonymousAccessDeniedRest() throws Exception { auditTrail.anonymousAccessDenied(request); if (includeRequestBody) { assertMsg(logger, Level.INFO, prefix + "[rest] [anonymous_access_denied]\torigin_address=[" + - NetworkAddress.format(address) + "], uri=[_uri], request_body=[" + expectedMessage + "]"); + NetworkAddress.format(address) + "], uri=[_uri]" + opaqueId + ", request_body=[" + expectedMessage + "]"); } else { assertMsg(logger, Level.INFO, prefix + "[rest] [anonymous_access_denied]\torigin_address=[" + - NetworkAddress.format(address) + "], uri=[_uri]"); + NetworkAddress.format(address) + "], uri=[_uri]" + opaqueId); } // test disabled @@ -202,10 +211,10 @@ public void testAuthenticationFailed() throws Exception { if (message instanceof IndicesRequest) { assertMsg(logger, Level.INFO, prefix + "[transport] [authentication_failed]\t" + origins + ", principal=[_principal], action=[_action], indices=[" + indices(message) + - "], request=[MockIndicesRequest]"); + "], request=[MockIndicesRequest]" + opaqueId); } else { assertMsg(logger, Level.INFO, prefix + "[transport] [authentication_failed]\t" + origins + - ", principal=[_principal], action=[_action], request=[MockMessage]"); + ", principal=[_principal], action=[_action], request=[MockMessage]" + opaqueId); } // test disabled @@ -224,10 +233,10 @@ public void testAuthenticationFailedNoToken() throws Exception { auditTrail.authenticationFailed("_action", message); if (message instanceof IndicesRequest) { assertMsg(logger, Level.INFO, prefix + "[transport] [authentication_failed]\t" + origins + - ", action=[_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]"); + ", action=[_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]" + opaqueId); } else { assertMsg(logger, Level.INFO, prefix + "[transport] [authentication_failed]\t" + origins + - ", action=[_action], request=[MockMessage]"); + ", action=[_action], request=[MockMessage]" + opaqueId); } // test disabled @@ -248,11 +257,11 @@ public void testAuthenticationFailedRest() throws Exception { auditTrail.authenticationFailed(new MockToken(), request); if (includeRequestBody) { assertMsg(logger, Level.INFO, prefix + "[rest] [authentication_failed]\torigin_address=[" + - NetworkAddress.format(address) + "], principal=[_principal], uri=[_uri], request_body=[" + + NetworkAddress.format(address) + "], principal=[_principal], uri=[_uri]" + opaqueId + ", request_body=[" + expectedMessage + "]"); } else { assertMsg(logger, Level.INFO, prefix + "[rest] [authentication_failed]\torigin_address=[" + - NetworkAddress.format(address) + "], principal=[_principal], uri=[_uri]"); + NetworkAddress.format(address) + "], principal=[_principal], uri=[_uri]" + opaqueId); } // test disabled @@ -273,10 +282,10 @@ public void testAuthenticationFailedRestNoToken() throws Exception { auditTrail.authenticationFailed(request); if (includeRequestBody) { assertMsg(logger, Level.INFO, prefix + "[rest] [authentication_failed]\torigin_address=[" + - NetworkAddress.format(address) + "], uri=[_uri], request_body=[" + expectedMessage + "]"); + NetworkAddress.format(address) + "], uri=[_uri]" + opaqueId + ", request_body=[" + expectedMessage + "]"); } else { assertMsg(logger, Level.INFO, prefix + "[rest] [authentication_failed]\torigin_address=[" + - NetworkAddress.format(address) + "], uri=[_uri]"); + NetworkAddress.format(address) + "], uri=[_uri]" + opaqueId); } // test disabled @@ -303,10 +312,10 @@ public void testAuthenticationFailedRealm() throws Exception { if (message instanceof IndicesRequest) { assertMsg(logger, Level.INFO, prefix + "[transport] [realm_authentication_failed]\trealm=[_realm], " + origins + ", principal=[_principal], action=[_action], indices=[" + indices(message) + "], " + - "request=[MockIndicesRequest]"); + "request=[MockIndicesRequest]" + opaqueId); } else { assertMsg(logger, Level.INFO, prefix + "[transport] [realm_authentication_failed]\trealm=[_realm], " + origins + - ", principal=[_principal], action=[_action], request=[MockMessage]"); + ", principal=[_principal], action=[_action], request=[MockMessage]" + opaqueId); } } @@ -327,11 +336,11 @@ public void testAuthenticationFailedRealmRest() throws Exception { auditTrail.authenticationFailed("_realm", new MockToken(), request); if (includeRequestBody) { assertMsg(logger, Level.INFO, prefix + "[rest] [realm_authentication_failed]\trealm=[_realm], origin_address=[" + - NetworkAddress.format(address) + "], principal=[_principal], uri=[_uri], request_body=[" + + NetworkAddress.format(address) + "], principal=[_principal], uri=[_uri]" + opaqueId + ", request_body=[" + expectedMessage + "]"); } else { assertMsg(logger, Level.INFO, prefix + "[rest] [realm_authentication_failed]\trealm=[_realm], origin_address=[" + - NetworkAddress.format(address) + "], principal=[_principal], uri=[_uri]"); + NetworkAddress.format(address) + "], principal=[_principal], uri=[_uri]" + opaqueId); } } @@ -353,10 +362,10 @@ public void testAccessGranted() throws Exception { : "principal=[_username], realm=[authRealm]") + ", roles=[" + role + "]"; if (message instanceof IndicesRequest) { assertMsg(logger, Level.INFO, prefix + "[transport] [access_granted]\t" + origins + ", " + userInfo + - ", action=[_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]"); + ", action=[_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]" + opaqueId); } else { assertMsg(logger, Level.INFO, prefix + "[transport] [access_granted]\t" + origins + ", " + userInfo + - ", action=[_action], request=[MockMessage]"); + ", action=[_action], request=[MockMessage]" + opaqueId); } // test disabled @@ -384,11 +393,11 @@ public void testAccessGrantedInternalSystemAction() throws Exception { assertMsg(logger, Level.INFO, prefix + "[transport] [access_granted]\t" + origins + ", principal=[" + SystemUser.INSTANCE.principal() + "], realm=[authRealm], roles=[" + role + "], action=[internal:_action], indices=[" + indices(message) - + "], request=[MockIndicesRequest]"); + + "], request=[MockIndicesRequest]" + opaqueId); } else { assertMsg(logger, Level.INFO, prefix + "[transport] [access_granted]\t" + origins + ", principal=[" + SystemUser.INSTANCE.principal() + "], realm=[authRealm], roles=[" + role - + "], action=[internal:_action], request=[MockMessage]"); + + "], action=[internal:_action], request=[MockMessage]" + opaqueId); } } @@ -410,10 +419,10 @@ public void testAccessGrantedInternalSystemActionNonSystemUser() throws Exceptio : "principal=[_username], realm=[authRealm]") + ", roles=[" + role + "]"; if (message instanceof IndicesRequest) { assertMsg(logger, Level.INFO, prefix + "[transport] [access_granted]\t" + origins + ", " + userInfo + - ", action=[internal:_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]"); + ", action=[internal:_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]" + opaqueId); } else { assertMsg(logger, Level.INFO, prefix + "[transport] [access_granted]\t" + origins + ", " + userInfo + - ", action=[internal:_action], request=[MockMessage]"); + ", action=[internal:_action], request=[MockMessage]" + opaqueId); } // test disabled @@ -442,10 +451,10 @@ public void testAccessDenied() throws Exception { : "principal=[_username], realm=[authRealm]") + ", roles=[" + role + "]"; if (message instanceof IndicesRequest) { assertMsg(logger, Level.INFO, prefix + "[transport] [access_denied]\t" + origins + ", " + userInfo + - ", action=[_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]"); + ", action=[_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]" + opaqueId); } else { assertMsg(logger, Level.INFO, prefix + "[transport] [access_denied]\t" + origins + ", " + userInfo + - ", action=[_action], request=[MockMessage]"); + ", action=[_action], request=[MockMessage]" + opaqueId); } // test disabled @@ -466,10 +475,10 @@ public void testTamperedRequestRest() throws Exception { auditTrail.tamperedRequest(request); if (includeRequestBody) { assertMsg(logger, Level.INFO, prefix + "[rest] [tampered_request]\torigin_address=[" + - NetworkAddress.format(address) + "], uri=[_uri], request_body=[" + expectedMessage + "]"); + NetworkAddress.format(address) + "], uri=[_uri]" + opaqueId + ", request_body=[" + expectedMessage + "]"); } else { assertMsg(logger, Level.INFO, prefix + "[rest] [tampered_request]\torigin_address=[" + - NetworkAddress.format(address) + "], uri=[_uri]"); + NetworkAddress.format(address) + "], uri=[_uri]" + opaqueId); } // test disabled @@ -489,10 +498,10 @@ public void testTamperedRequest() throws Exception { auditTrail.tamperedRequest(action, message); if (message instanceof IndicesRequest) { assertMsg(logger, Level.INFO, prefix + "[transport] [tampered_request]\t" + origins + - ", action=[_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]"); + ", action=[_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]" + opaqueId); } else { assertMsg(logger, Level.INFO, prefix + "[transport] [tampered_request]\t" + origins + - ", action=[_action], request=[MockMessage]"); + ", action=[_action], request=[MockMessage]" + opaqueId); } // test disabled @@ -516,10 +525,10 @@ public void testTamperedRequestWithUser() throws Exception { auditTrail.tamperedRequest(user, action, message); if (message instanceof IndicesRequest) { assertMsg(logger, Level.INFO, prefix + "[transport] [tampered_request]\t" + origins + ", " + userInfo + - ", action=[_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]"); + ", action=[_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]" + opaqueId); } else { assertMsg(logger, Level.INFO, prefix + "[transport] [tampered_request]\t" + origins + ", " + userInfo + - ", action=[_action], request=[MockMessage]"); + ", action=[_action], request=[MockMessage]" + opaqueId); } // test disabled @@ -537,7 +546,7 @@ public void testConnectionDenied() throws Exception { final SecurityIpFilterRule rule = new SecurityIpFilterRule(false, "_all"); auditTrail.connectionDenied(inetAddress, "default", rule); assertMsg(logger, Level.INFO, String.format(Locale.ROOT, prefix + - "[ip_filter] [connection_denied]\torigin_address=[%s], transport_profile=[%s], rule=[deny %s]", + "[ip_filter] [connection_denied]\torigin_address=[%s], transport_profile=[%s], rule=[deny %s]" + opaqueId, NetworkAddress.format(inetAddress), "default", "_all")); // test disabled @@ -562,7 +571,8 @@ public void testConnectionGranted() throws Exception { auditTrail = new LoggingAuditTrail(settings, clusterService, logger, threadContext); auditTrail.connectionGranted(inetAddress, "default", rule); assertMsg(logger, Level.INFO, String.format(Locale.ROOT, prefix + "[ip_filter] [connection_granted]\torigin_address=[%s], " + - "transport_profile=[default], rule=[allow default:accept_all]", NetworkAddress.format(inetAddress))); + "transport_profile=[default], rule=[allow default:accept_all]" + opaqueId, + NetworkAddress.format(inetAddress))); } public void testRunAsGranted() throws Exception { @@ -577,12 +587,12 @@ public void testRunAsGranted() throws Exception { assertMsg(logger, Level.INFO, prefix + "[transport] [run_as_granted]\t" + origins + ", principal=[_username], realm=[authRealm], run_as_principal=[running as], run_as_realm=[lookRealm], roles=[" - + role + "], action=[_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]"); + + role + "], action=[_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]" + opaqueId); } else { assertMsg(logger, Level.INFO, prefix + "[transport] [run_as_granted]\t" + origins + ", principal=[_username], realm=[authRealm], run_as_principal=[running as], run_as_realm=[lookRealm], roles=[" - + role + "], action=[_action], request=[MockMessage]"); + + role + "], action=[_action], request=[MockMessage]" + opaqueId); } // test disabled @@ -605,12 +615,12 @@ public void testRunAsDenied() throws Exception { assertMsg(logger, Level.INFO, prefix + "[transport] [run_as_denied]\t" + origins + ", principal=[_username], realm=[authRealm], run_as_principal=[running as], run_as_realm=[lookRealm], roles=[" - + role + "], action=[_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]"); + + role + "], action=[_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]" + opaqueId); } else { assertMsg(logger, Level.INFO, prefix + "[transport] [run_as_denied]\t" + origins + ", principal=[_username], realm=[authRealm], run_as_principal=[running as], run_as_realm=[lookRealm], roles=[" - + role + "], action=[_action], request=[MockMessage]"); + + role + "], action=[_action], request=[MockMessage]" + opaqueId); } // test disabled @@ -667,10 +677,11 @@ public void testAuthenticationSuccessRest() throws Exception { if (includeRequestBody) { assertMsg(logger, Level.INFO, prefix + "[rest] [authentication_success]\t" + userInfo + ", realm=[_realm], uri=[_uri], params=[" + params - + "], request_body=[" + expectedMessage + "]"); + + "]" + opaqueId + ", request_body=[" + expectedMessage + "]"); } else { assertMsg(logger, Level.INFO, - prefix + "[rest] [authentication_success]\t" + userInfo + ", realm=[_realm], uri=[_uri], params=[" + params + "]"); + prefix + "[rest] [authentication_success]\t" + userInfo + ", realm=[_realm], uri=[_uri], params=[" + params + + "]" + opaqueId); } // test disabled @@ -701,10 +712,10 @@ public void testAuthenticationSuccessTransport() throws Exception { auditTrail.authenticationSuccess(realm, user, "_action", message); if (message instanceof IndicesRequest) { assertMsg(logger, Level.INFO, prefix + "[transport] [authentication_success]\t" + origins + ", " + userInfo - + ", realm=[_realm], action=[_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]"); + + ", realm=[_realm], action=[_action], indices=[" + indices(message) + "], request=[MockIndicesRequest]" + opaqueId); } else { assertMsg(logger, Level.INFO, prefix + "[transport] [authentication_success]\t" + origins + ", " + userInfo - + ", realm=[_realm], action=[_action], request=[MockMessage]"); + + ", realm=[_realm], action=[_action], request=[MockMessage]" + opaqueId); } // test disabled From 74fddb3fecbca96c1040c65c7c4cd1849538635f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 10 Jul 2018 09:19:11 +0200 Subject: [PATCH 12/50] Change trappy float comparison (#31889) Comparing primitive floats with '==' can be trappy because e.g. special handling of NaN values. It can be avoided by using Float.compare(). --- .../routing/allocation/allocator/BalancedShardsAllocator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java index 7998a1d27dd66..ec0af211ecca5 100644 --- a/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java +++ b/server/src/main/java/org/elasticsearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java @@ -1009,7 +1009,7 @@ private boolean tryRelocateShard(ModelNode minNode, ModelNode maxNode, String id // simulate moving shard from maxNode to minNode final float delta = weight.weightShardAdded(this, minNode, idx) - weight.weightShardRemoved(this, maxNode, idx); if (delta < minCost || - (candidate != null && delta == minCost && candidate.id() > shard.id())) { + (candidate != null && Float.compare(delta, minCost) == 0 && candidate.id() > shard.id())) { /* this last line is a tie-breaker to make the shard allocation alg deterministic * otherwise we rely on the iteration order of the index.getAllShards() which is a set.*/ minCost = delta; From 284a20afddc55cbe8ebe890b9885790226c0eff8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 10 Jul 2018 09:34:17 +0200 Subject: [PATCH 13/50] Fix broken NaN check in MovingFunctions#stdDev() (#31888) The initial check will never be true, because of the special semantics of NaN, where no value is equal to Nan, including NaN. Thus, x == Double.NaN always evaluates to false. The method still works correct because later computations will also return NaN if the avg argument is NaN, but the intended shortcut doesn't work. --- .../search/aggregations/pipeline/movfn/MovingFunctions.java | 2 +- .../pipeline/movfn/MovFnWhitelistedFunctionTests.java | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/movfn/MovingFunctions.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/movfn/MovingFunctions.java index c38ce27cd8ad2..020189d461935 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/movfn/MovingFunctions.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/movfn/MovingFunctions.java @@ -85,7 +85,7 @@ public static double unweightedAvg(double[] values) { * The average is based on the count of non-null, non-NaN values. */ public static double stdDev(double[] values, double avg) { - if (avg == Double.NaN) { + if (Double.isNaN(avg)) { return Double.NaN; } else { long count = 0; diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/movfn/MovFnWhitelistedFunctionTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/movfn/MovFnWhitelistedFunctionTests.java index 0a0f9d6ae3759..6d0e388e64325 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/movfn/MovFnWhitelistedFunctionTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/movfn/MovFnWhitelistedFunctionTests.java @@ -313,6 +313,10 @@ public void testEmptySimpleStdDev() { assertThat(actual, equalTo(Double.NaN)); } + public void testStdDevNaNAvg() { + assertThat(MovingFunctions.stdDev(new double[] { 1.0, 2.0, 3.0 }, Double.NaN), equalTo(Double.NaN)); + } + public void testLinearMovAvg() { int numValues = randomIntBetween(1, 100); From 4424bfc91c75618ea66a79e1698afe336afe7eaf Mon Sep 17 00:00:00 2001 From: Tim Vernum Date: Tue, 10 Jul 2018 11:54:24 +1000 Subject: [PATCH 14/50] Fix building AD URL from domain name (#31849) The steps to read the settings and build URLs happen in a non-obvious order, which meant that we would build the default URL (from the domain name, and port) before we'd actually read the port settings. This would cause the URL to always have a port of `0`. Relates: bccf988 --- .../ldap/ActiveDirectorySessionFactory.java | 12 +++-- .../authc/ldap/ActiveDirectoryRealmTests.java | 47 +++++++++++++++++++ 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactory.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactory.java index 295e00e19a8a9..d175e1b229312 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactory.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectorySessionFactory.java @@ -11,6 +11,7 @@ import com.unboundid.ldap.sdk.LDAPException; import com.unboundid.ldap.sdk.LDAPInterface; import com.unboundid.ldap.sdk.SearchResultEntry; +import com.unboundid.ldap.sdk.ServerSet; import com.unboundid.ldap.sdk.SimpleBindRequest; import com.unboundid.ldap.sdk.controls.AuthorizationIdentityRequestControl; import org.apache.logging.log4j.Logger; @@ -62,8 +63,6 @@ class ActiveDirectorySessionFactory extends PoolingSessionFactory { final DownLevelADAuthenticator downLevelADAuthenticator; final UpnADAuthenticator upnADAuthenticator; - private final int ldapPort; - ActiveDirectorySessionFactory(RealmConfig config, SSLService sslService, ThreadPool threadPool) throws LDAPException { super(config, sslService, new ActiveDirectoryGroupsResolver(config.settings()), ActiveDirectorySessionFactorySettings.POOL_ENABLED, @@ -85,7 +84,7 @@ class ActiveDirectorySessionFactory extends PoolingSessionFactory { + "] setting for active directory"); } String domainDN = buildDnFromDomain(domainName); - ldapPort = ActiveDirectorySessionFactorySettings.AD_LDAP_PORT_SETTING.get(settings); + final int ldapPort = ActiveDirectorySessionFactorySettings.AD_LDAP_PORT_SETTING.get(settings); final int ldapsPort = ActiveDirectorySessionFactorySettings.AD_LDAPS_PORT_SETTING.get(settings); final int gcLdapPort = ActiveDirectorySessionFactorySettings.AD_GC_LDAP_PORT_SETTING.get(settings); final int gcLdapsPort = ActiveDirectorySessionFactorySettings.AD_GC_LDAPS_PORT_SETTING.get(settings); @@ -102,7 +101,7 @@ class ActiveDirectorySessionFactory extends PoolingSessionFactory { @Override protected List getDefaultLdapUrls(Settings settings) { return Collections.singletonList("ldap://" + settings.get(ActiveDirectorySessionFactorySettings.AD_DOMAIN_NAME_SETTING) + - ":" + ldapPort); + ":" + ActiveDirectorySessionFactorySettings.AD_LDAP_PORT_SETTING.get(settings)); } @Override @@ -197,6 +196,11 @@ static String getBindDN(Settings settings) { return bindDN; } + // Exposed for testing + ServerSet getServerSet() { + return super.serverSet; + } + ADAuthenticator getADAuthenticator(String username) { if (username.indexOf('\\') > 0) { return downLevelADAuthenticator; diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java index bcd7996e32a8c..739523795e7c5 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authc/ldap/ActiveDirectoryRealmTests.java @@ -8,8 +8,10 @@ import com.unboundid.ldap.listener.InMemoryDirectoryServer; import com.unboundid.ldap.listener.InMemoryDirectoryServerConfig; import com.unboundid.ldap.sdk.Attribute; +import com.unboundid.ldap.sdk.FailoverServerSet; import com.unboundid.ldap.sdk.LDAPException; import com.unboundid.ldap.sdk.LDAPURL; +import com.unboundid.ldap.sdk.SingleServerSet; import com.unboundid.ldap.sdk.schema.Schema; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.support.PlainActionFuture; @@ -28,6 +30,7 @@ import org.elasticsearch.xpack.core.security.authc.ldap.ActiveDirectorySessionFactorySettings; import org.elasticsearch.xpack.core.security.authc.ldap.LdapRealmSettings; import org.elasticsearch.xpack.core.security.authc.ldap.PoolingSessionFactorySettings; +import org.elasticsearch.xpack.core.security.authc.ldap.support.SessionFactorySettings; import org.elasticsearch.xpack.core.security.authc.support.CachingUsernamePasswordRealmSettings; import org.elasticsearch.xpack.core.security.authc.support.DnRoleMapperSettings; import org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken; @@ -51,9 +54,11 @@ import static org.elasticsearch.xpack.core.security.authc.ldap.support.SessionFactorySettings.URLS_SETTING; import static org.hamcrest.Matchers.arrayContaining; import static org.hamcrest.Matchers.arrayContainingInAnyOrder; +import static org.hamcrest.Matchers.arrayWithSize; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.mockito.Matchers.any; @@ -355,6 +360,48 @@ public void testCustomSearchFilters() throws Exception { assertEquals("(objectClass=down level)", sessionFactory.downLevelADAuthenticator.getUserSearchFilter()); } + public void testBuildUrlFromDomainNameAndDefaultPort() throws Exception { + Settings settings = Settings.builder() + .put(ActiveDirectorySessionFactorySettings.AD_DOMAIN_NAME_SETTING, "ad.test.elasticsearch.com") + .build(); + RealmConfig config = new RealmConfig("testBuildUrlFromDomainNameAndDefaultPort", settings, globalSettings, + TestEnvironment.newEnvironment(globalSettings), new ThreadContext(globalSettings)); + ActiveDirectorySessionFactory sessionFactory = new ActiveDirectorySessionFactory(config, sslService, threadPool); + assertSingleLdapServer(sessionFactory, "ad.test.elasticsearch.com", 389); + } + + public void testBuildUrlFromDomainNameAndCustomPort() throws Exception { + Settings settings = Settings.builder() + .put(ActiveDirectorySessionFactorySettings.AD_DOMAIN_NAME_SETTING, "ad.test.elasticsearch.com") + .put(ActiveDirectorySessionFactorySettings.AD_LDAP_PORT_SETTING.getKey(), 10389) + .build(); + RealmConfig config = new RealmConfig("testBuildUrlFromDomainNameAndCustomPort", settings, globalSettings, + TestEnvironment.newEnvironment(globalSettings), new ThreadContext(globalSettings)); + ActiveDirectorySessionFactory sessionFactory = new ActiveDirectorySessionFactory(config, sslService, threadPool); + assertSingleLdapServer(sessionFactory, "ad.test.elasticsearch.com", 10389); + } + + public void testUrlConfiguredInSettings() throws Exception { + Settings settings = Settings.builder() + .put(ActiveDirectorySessionFactorySettings.AD_DOMAIN_NAME_SETTING, "ad.test.elasticsearch.com") + .put(SessionFactorySettings.URLS_SETTING, "ldap://ad01.testing.elastic.co:20389/") + .build(); + RealmConfig config = new RealmConfig("testBuildUrlFromDomainNameAndCustomPort", settings, globalSettings, + TestEnvironment.newEnvironment(globalSettings), new ThreadContext(globalSettings)); + ActiveDirectorySessionFactory sessionFactory = new ActiveDirectorySessionFactory(config, sslService, threadPool); + assertSingleLdapServer(sessionFactory, "ad01.testing.elastic.co", 20389); + } + + private void assertSingleLdapServer(ActiveDirectorySessionFactory sessionFactory, String hostname, int port) { + assertThat(sessionFactory.getServerSet(), instanceOf(FailoverServerSet.class)); + FailoverServerSet fss = (FailoverServerSet) sessionFactory.getServerSet(); + assertThat(fss.getServerSets(), arrayWithSize(1)); + assertThat(fss.getServerSets()[0], instanceOf(SingleServerSet.class)); + SingleServerSet sss = (SingleServerSet) fss.getServerSets()[0]; + assertThat(sss.getAddress(), equalTo(hostname)); + assertThat(sss.getPort(), equalTo(port)); + } + private Settings settings() throws Exception { return settings(Settings.EMPTY); } From eb30825524552368e2dffd6e32b60c71fe6c3bfa Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Tue, 10 Jul 2018 13:12:15 +0200 Subject: [PATCH 15/50] Backport: Add template config for Beat state to X-Pack Monitoring (#31809) (#31893) With https://github.com/elastic/beats/pull/7075 Beats introduces state reporting for X-Pack Monitoring. The data sent up to Elasticsearch ends up stored in the following format. ``` "beats_state": { "timestamp": "2018-07-05T07:21:03.581Z", "state": { "module": { "count": 1, "names": [ "http" ] } }, "beat": { "uuid": "594039b5-6353-4d78-9bad-778ecc0fe83f", "type": "metricbeat", "version": "7.0.0-alpha1", "name": "ruflin", "host": "ruflin" } } ``` This PR adds the new fields to the template. --- .../src/main/resources/monitoring-beats.json | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/x-pack/plugin/core/src/main/resources/monitoring-beats.json b/x-pack/plugin/core/src/main/resources/monitoring-beats.json index 28f312c7dcbb6..ff9e79525143a 100644 --- a/x-pack/plugin/core/src/main/resources/monitoring-beats.json +++ b/x-pack/plugin/core/src/main/resources/monitoring-beats.json @@ -12,6 +12,47 @@ "doc": { "dynamic": false, "properties": { + "beats_state": { + "properties": { + "beat": { + "properties": { + "host": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "uuid": { + "type": "keyword" + }, + "version": { + "type": "keyword" + } + } + }, + "state": { + "properties": { + "module": { + "properties": { + "count": { + "type": "long" + }, + "names": { + "type": "keyword" + } + } + } + } + }, + "timestamp": { + "format": "date_time", + "type": "date" + } + } + }, "beats_stats": { "properties": { "beat": { From f17691da210a2920240eb290ef0c27ebc4fb30cb Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Tue, 10 Jul 2018 13:21:56 -0400 Subject: [PATCH 16/50] Increase logging level for :qa:rolling-upgrade Hopefully, we can capture the source of testHistoryUUIDIsGenerated. --- qa/rolling-upgrade/build.gradle | 2 +- .../src/main/java/org/elasticsearch/index/store/Store.java | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/qa/rolling-upgrade/build.gradle b/qa/rolling-upgrade/build.gradle index 76fb4de0037ab..83440ceeeb602 100644 --- a/qa/rolling-upgrade/build.gradle +++ b/qa/rolling-upgrade/build.gradle @@ -64,7 +64,6 @@ for (Version version : bwcVersions.wireCompatible) { clusterName = 'rolling-upgrade' setting 'repositories.url.allowed_urls', 'http://snapshot.test*' setting 'node.attr.gen', 'old' - // debug logging for testHistoryUUIDIsGenerated setting 'logger.level', 'DEBUG' if (version.onOrAfter('5.3.0')) { setting 'http.content_type.required', 'true' @@ -88,6 +87,7 @@ for (Version version : bwcVersions.wireCompatible) { dataDir = { nodeNumber -> oldClusterTest.nodes[stopNode].dataDir } setting 'repositories.url.allowed_urls', 'http://snapshot.test*' setting 'node.name', "upgraded-node-${stopNode}" + setting 'logger.level', 'DEBUG' } } diff --git a/server/src/main/java/org/elasticsearch/index/store/Store.java b/server/src/main/java/org/elasticsearch/index/store/Store.java index 545c335e3a2da..03b641b69ce57 100644 --- a/server/src/main/java/org/elasticsearch/index/store/Store.java +++ b/server/src/main/java/org/elasticsearch/index/store/Store.java @@ -1461,7 +1461,7 @@ public void bootstrapNewHistory() throws IOException { map.put(Engine.HISTORY_UUID_KEY, UUIDs.randomBase64UUID()); map.put(SequenceNumbers.MAX_SEQ_NO, Long.toString(seqno.maxSeqNo)); map.put(SequenceNumbers.LOCAL_CHECKPOINT_KEY, Long.toString(seqno.maxSeqNo)); - logger.debug("bootstrap a new history_uuid [{}]", map); + logger.debug("bootstrap a new history_uuid [{}], user_data [{}]", map, userData); updateCommitData(writer, map); } finally { metadataLock.writeLock().unlock(); @@ -1513,7 +1513,7 @@ public boolean ensureIndexHas6xCommitTags() throws IOException { maps.put(InternalEngine.MAX_UNSAFE_AUTO_ID_TIMESTAMP_COMMIT_ID, "-1"); } if (maps.isEmpty() == false) { - logger.debug("bootstrap 6.x commit tags [{}]", maps); + logger.debug("bootstrap 6.x commit tags [{}], user_data [{}]", maps, userData); updateCommitData(writer, maps); return true; } @@ -1584,6 +1584,7 @@ public void trimUnsafeCommits(final long lastSyncedGlobalCheckpoint, final long + startingIndexCommit.getUserData().get(Translog.TRANSLOG_UUID_KEY) + "] is not equal to last commit's translog uuid [" + translogUUID + "]"); } + logger.debug("starting index commit [{}]", startingIndexCommit.getUserData()); if (startingIndexCommit.equals(existingCommits.get(existingCommits.size() - 1)) == false) { try (IndexWriter writer = newIndexWriter(IndexWriterConfig.OpenMode.APPEND, directory, startingIndexCommit)) { // this achieves two things: From 1f090d3e1c903316c5d5784894cd940f2b29302b Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 10 Jul 2018 13:01:28 -0400 Subject: [PATCH 17/50] HLRest: Move xPackInfo() to xPack().info() (#31905) Originally I put the X-Pack info object into the top level rest client object. I did that because we thought we'd like to squash `xpack` from the name of the X-Pack APIs now that it is part of the default distribution. We still kind of want to do that, but at least for now we feel like it is better to keep the high level rest client aligned with the other language clients like C# and Python. This shifts the X-Pack info API to align with its json spec file. Relates to #31870 --- .../client/RestHighLevelClient.java | 44 ++++------- .../org/elasticsearch/client/XPackClient.java | 73 +++++++++++++++++++ .../elasticsearch/client/PingAndInfoIT.java | 6 +- .../MiscellaneousDocumentationIT.java | 4 +- .../org/elasticsearch/license/License.java | 2 +- .../action/TransportXPackInfoActionTests.java | 2 +- .../ml/datafeed/MlRemoteLicenseChecker.java | 2 +- .../datafeed/MlRemoteLicenseCheckerTests.java | 2 +- .../protocol/xpack/XPackInfoResponse.java | 2 +- .../{ => xpack}/license/LicenseStatus.java | 2 +- .../{ => xpack}/license/package-info.java | 2 +- .../{ => xpack}/security/package-info.java | 2 +- .../{ => xpack}/watcher/package-info.java | 2 +- .../xpack/XPackInfoResponseTests.java | 2 +- .../license/LicenseStatusTests.java | 2 +- 15 files changed, 103 insertions(+), 46 deletions(-) create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/XPackClient.java rename x-pack/protocol/src/main/java/org/elasticsearch/protocol/{ => xpack}/license/LicenseStatus.java (97%) rename x-pack/protocol/src/main/java/org/elasticsearch/protocol/{ => xpack}/license/package-info.java (94%) rename x-pack/protocol/src/main/java/org/elasticsearch/protocol/{ => xpack}/security/package-info.java (94%) rename x-pack/protocol/src/main/java/org/elasticsearch/protocol/{ => xpack}/watcher/package-info.java (94%) rename x-pack/protocol/src/test/java/org/elasticsearch/protocol/{ => xpack}/license/LicenseStatusTests.java (95%) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java index bff43bdd6e3b2..736a49879188b 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java @@ -66,8 +66,6 @@ import org.elasticsearch.index.rankeval.RankEvalRequest; import org.elasticsearch.index.rankeval.RankEvalResponse; import org.elasticsearch.plugins.spi.NamedXContentProvider; -import org.elasticsearch.protocol.xpack.XPackInfoRequest; -import org.elasticsearch.protocol.xpack.XPackInfoResponse; import org.elasticsearch.rest.BytesRestResponse; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.script.mustache.MultiSearchTemplateRequest; @@ -204,6 +202,7 @@ public class RestHighLevelClient implements Closeable { private final IngestClient ingestClient = new IngestClient(this); private final SnapshotClient snapshotClient = new SnapshotClient(this); private final TasksClient tasksClient = new TasksClient(this); + private final XPackClient xPackClient = new XPackClient(this); /** * Creates a {@link RestHighLevelClient} given the low level {@link RestClientBuilder} that allows to build the @@ -294,6 +293,19 @@ public final TasksClient tasks() { return tasksClient; } + /** + * A wrapper for the {@link RestHighLevelClient} that provides methods for + * accessing the Elastic Licensed X-Pack APIs that are shipped with the + * default distribution of Elasticsearch. All of these APIs will 404 if run + * against the OSS distribution of Elasticsearch. + *

    + * See the + * X-Pack APIs on elastic.co for more information. + */ + public final XPackClient xpack() { + return xPackClient; + } + /** * Executes a bulk request using the Bulk API. * See Bulk API on elastic.co @@ -1104,34 +1116,6 @@ public final void fieldCapsAsync(FieldCapabilitiesRequest fieldCapabilitiesReque FieldCapabilitiesResponse::fromXContent, listener, emptySet()); } - /** - * Fetch information about X-Pack from the cluster if it is installed. - * See - * the docs for more. - * @param request the request - * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized - * @return the response - * @throws IOException in case there is a problem sending the request or parsing back the response - */ - public XPackInfoResponse xPackInfo(XPackInfoRequest request, RequestOptions options) throws IOException { - return performRequestAndParseEntity(request, RequestConverters::xPackInfo, options, - XPackInfoResponse::fromXContent, emptySet()); - } - - /** - * Fetch information about X-Pack from the cluster if it is installed. - * See - * the docs for more. - * @param request the request - * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized - * @param listener the listener to be notified upon request completion - */ - public void xPackInfoAsync(XPackInfoRequest request, RequestOptions options, - ActionListener listener) { - performRequestAsyncAndParseEntity(request, RequestConverters::xPackInfo, options, - XPackInfoResponse::fromXContent, listener, emptySet()); - } - @Deprecated protected final Resp performRequestAndParseEntity(Req request, CheckedFunction requestConverter, diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/XPackClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/XPackClient.java new file mode 100644 index 0000000000000..5942bfa35a477 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/XPackClient.java @@ -0,0 +1,73 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.protocol.xpack.XPackInfoRequest; +import org.elasticsearch.protocol.xpack.XPackInfoResponse; + +import java.io.IOException; + +import static java.util.Collections.emptySet; + +/** + * A wrapper for the {@link RestHighLevelClient} that provides methods for + * accessing the Elastic Licensed X-Pack APIs that are shipped with the + * default distribution of Elasticsearch. All of these APIs will 404 if run + * against the OSS distribution of Elasticsearch. + *

    + * See the + * X-Pack APIs on elastic.co for more information. + */ +public final class XPackClient { + private final RestHighLevelClient restHighLevelClient; + + XPackClient(RestHighLevelClient restHighLevelClient) { + this.restHighLevelClient = restHighLevelClient; + } + + /** + * Fetch information about X-Pack from the cluster. + * See + * the docs for more. + * @param request the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public XPackInfoResponse info(XPackInfoRequest request, RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(request, RequestConverters::xPackInfo, options, + XPackInfoResponse::fromXContent, emptySet()); + } + + /** + * Asynchronously fetch information about X-Pack from the cluster. + * See + * the docs for more. + * @param request the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public void infoAsync(XPackInfoRequest request, RequestOptions options, + ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(request, RequestConverters::xPackInfo, options, + XPackInfoResponse::fromXContent, listener, emptySet()); + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/PingAndInfoIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/PingAndInfoIT.java index 04bac628f4930..1a50187f5df5d 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/PingAndInfoIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/PingAndInfoIT.java @@ -21,10 +21,10 @@ import org.apache.http.client.methods.HttpGet; import org.elasticsearch.action.main.MainResponse; -import org.elasticsearch.protocol.license.LicenseStatus; import org.elasticsearch.protocol.xpack.XPackInfoRequest; import org.elasticsearch.protocol.xpack.XPackInfoResponse; import org.elasticsearch.protocol.xpack.XPackInfoResponse.FeatureSetsInfo.FeatureSet; +import org.elasticsearch.protocol.xpack.license.LicenseStatus; import java.io.IOException; import java.util.EnumSet; @@ -60,7 +60,7 @@ public void testXPackInfo() throws IOException { XPackInfoRequest request = new XPackInfoRequest(); request.setCategories(EnumSet.allOf(XPackInfoRequest.Category.class)); request.setVerbose(true); - XPackInfoResponse info = highLevelClient().xPackInfo(request, RequestOptions.DEFAULT); + XPackInfoResponse info = highLevelClient().xpack().info(request, RequestOptions.DEFAULT); MainResponse mainResponse = highLevelClient().info(RequestOptions.DEFAULT); @@ -89,7 +89,7 @@ public void testXPackInfo() throws IOException { } public void testXPackInfoEmptyRequest() throws IOException { - XPackInfoResponse info = highLevelClient().xPackInfo(new XPackInfoRequest(), RequestOptions.DEFAULT); + XPackInfoResponse info = highLevelClient().xpack().info(new XPackInfoRequest(), RequestOptions.DEFAULT); /* * The default in the transport client is non-verbose and returning diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MiscellaneousDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MiscellaneousDocumentationIT.java index 1729e6b22aa5b..639a5ce7ceef6 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MiscellaneousDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/MiscellaneousDocumentationIT.java @@ -87,7 +87,7 @@ public void testXPackInfo() throws Exception { XPackInfoRequest.Category.BUILD, XPackInfoRequest.Category.LICENSE, XPackInfoRequest.Category.FEATURES)); - XPackInfoResponse response = client.xPackInfo(request, RequestOptions.DEFAULT); + XPackInfoResponse response = client.xpack().info(request, RequestOptions.DEFAULT); //end::x-pack-info-execute //tag::x-pack-info-response @@ -123,7 +123,7 @@ public void onFailure(Exception e) { listener = new LatchedActionListener<>(listener, latch); // tag::x-pack-info-execute-async - client.xPackInfoAsync(request, RequestOptions.DEFAULT, listener); // <1> + client.xpack().infoAsync(request, RequestOptions.DEFAULT, listener); // <1> // end::x-pack-info-execute-async assertTrue(latch.await(30L, TimeUnit.SECONDS)); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/License.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/License.java index 78a66c99323d0..9b8f5e3cc00ec 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/License.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/License.java @@ -29,7 +29,7 @@ import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.protocol.license.LicenseStatus; +import org.elasticsearch.protocol.xpack.license.LicenseStatus; /** * Data structure for license. Use {@link Builder} to build a license. diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/TransportXPackInfoActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/TransportXPackInfoActionTests.java index 459f7fab02d72..4a8b1c051873e 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/TransportXPackInfoActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/action/TransportXPackInfoActionTests.java @@ -11,10 +11,10 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.license.License; import org.elasticsearch.license.LicenseService; -import org.elasticsearch.protocol.license.LicenseStatus; import org.elasticsearch.protocol.xpack.XPackInfoRequest; import org.elasticsearch.protocol.xpack.XPackInfoResponse; import org.elasticsearch.protocol.xpack.XPackInfoResponse.FeatureSetsInfo.FeatureSet; +import org.elasticsearch.protocol.xpack.license.LicenseStatus; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.Transport; diff --git a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/MlRemoteLicenseChecker.java b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/MlRemoteLicenseChecker.java index 791be9b7cb430..b0eeed2c800ec 100644 --- a/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/MlRemoteLicenseChecker.java +++ b/x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/datafeed/MlRemoteLicenseChecker.java @@ -13,9 +13,9 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.license.License; -import org.elasticsearch.protocol.license.LicenseStatus; import org.elasticsearch.protocol.xpack.XPackInfoRequest; import org.elasticsearch.protocol.xpack.XPackInfoResponse; +import org.elasticsearch.protocol.xpack.license.LicenseStatus; import org.elasticsearch.transport.ActionNotFoundTransportException; import org.elasticsearch.transport.RemoteClusterAware; import org.elasticsearch.xpack.core.action.XPackInfoAction; diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/MlRemoteLicenseCheckerTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/MlRemoteLicenseCheckerTests.java index c10bf1081dd80..81e4c75cfad7c 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/MlRemoteLicenseCheckerTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/MlRemoteLicenseCheckerTests.java @@ -11,8 +11,8 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; -import org.elasticsearch.protocol.license.LicenseStatus; import org.elasticsearch.protocol.xpack.XPackInfoResponse; +import org.elasticsearch.protocol.xpack.license.LicenseStatus; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.xpack.core.action.XPackInfoAction; diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoResponse.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoResponse.java index d38174e03f3a4..3b9032f092185 100644 --- a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoResponse.java +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/XPackInfoResponse.java @@ -31,7 +31,7 @@ import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.protocol.license.LicenseStatus; +import org.elasticsearch.protocol.xpack.license.LicenseStatus; import java.io.IOException; import java.util.ArrayList; diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/license/LicenseStatus.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/license/LicenseStatus.java similarity index 97% rename from x-pack/protocol/src/main/java/org/elasticsearch/protocol/license/LicenseStatus.java rename to x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/license/LicenseStatus.java index f0f49351ab1d8..ea3e4f8a8965b 100644 --- a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/license/LicenseStatus.java +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/license/LicenseStatus.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.protocol.license; +package org.elasticsearch.protocol.xpack.license; import java.io.IOException; diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/license/package-info.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/license/package-info.java similarity index 94% rename from x-pack/protocol/src/main/java/org/elasticsearch/protocol/license/package-info.java rename to x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/license/package-info.java index f671b280d8495..ca859f29e440b 100644 --- a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/license/package-info.java +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/license/package-info.java @@ -21,4 +21,4 @@ * Request and Response objects for the default distribution's License * APIs. */ -package org.elasticsearch.protocol.license; +package org.elasticsearch.protocol.xpack.license; diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/security/package-info.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/security/package-info.java similarity index 94% rename from x-pack/protocol/src/main/java/org/elasticsearch/protocol/security/package-info.java rename to x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/security/package-info.java index 2a0ed5bc52b15..216990d9f0ec0 100644 --- a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/security/package-info.java +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/security/package-info.java @@ -21,4 +21,4 @@ * Request and Response objects for the default distribution's Security * APIs. */ -package org.elasticsearch.protocol.security; +package org.elasticsearch.protocol.xpack.security; diff --git a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/watcher/package-info.java b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/watcher/package-info.java similarity index 94% rename from x-pack/protocol/src/main/java/org/elasticsearch/protocol/watcher/package-info.java rename to x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/watcher/package-info.java index 43d7dd29ec9e5..d34fd598ab170 100644 --- a/x-pack/protocol/src/main/java/org/elasticsearch/protocol/watcher/package-info.java +++ b/x-pack/protocol/src/main/java/org/elasticsearch/protocol/xpack/watcher/package-info.java @@ -21,4 +21,4 @@ * Request and Response objects for the default distribution's Watcher * APIs. */ -package org.elasticsearch.protocol.watcher; +package org.elasticsearch.protocol.xpack.watcher; diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/XPackInfoResponseTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/XPackInfoResponseTests.java index 61a936b0df880..820b8200ed940 100644 --- a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/XPackInfoResponseTests.java +++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/XPackInfoResponseTests.java @@ -21,11 +21,11 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.protocol.license.LicenseStatus; import org.elasticsearch.protocol.xpack.XPackInfoResponse.BuildInfo; import org.elasticsearch.protocol.xpack.XPackInfoResponse.LicenseInfo; import org.elasticsearch.protocol.xpack.XPackInfoResponse.FeatureSetsInfo; import org.elasticsearch.protocol.xpack.XPackInfoResponse.FeatureSetsInfo.FeatureSet; +import org.elasticsearch.protocol.xpack.license.LicenseStatus; import org.elasticsearch.test.AbstractStreamableXContentTestCase; import java.util.HashMap; diff --git a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/license/LicenseStatusTests.java b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/license/LicenseStatusTests.java similarity index 95% rename from x-pack/protocol/src/test/java/org/elasticsearch/protocol/license/LicenseStatusTests.java rename to x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/license/LicenseStatusTests.java index 2c87645157c78..c256e7562f7f8 100644 --- a/x-pack/protocol/src/test/java/org/elasticsearch/protocol/license/LicenseStatusTests.java +++ b/x-pack/protocol/src/test/java/org/elasticsearch/protocol/xpack/license/LicenseStatusTests.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package org.elasticsearch.protocol.license; +package org.elasticsearch.protocol.xpack.license; import java.io.IOException; From 4de66ba7048905183562d326142d7349282d1190 Mon Sep 17 00:00:00 2001 From: Andy Bristol Date: Tue, 10 Jul 2018 13:36:30 -0700 Subject: [PATCH 18/50] [test] port archive distribution packaging tests (#31314) Recreates the rest of the bats packaging tests for the tar distribution in the java packaging test project, with support for both tar and zip packaging, both oss and default flavors, and on Linux and Windows. Most tests are followed fairly closely, some have either been dropped if unnecessary or folded into others if convenient. --- .../gradle/vagrant/VagrantTestPlugin.groovy | 10 +- qa/vagrant/build.gradle | 20 ++ .../packaging/test/ArchiveTestCase.java | 252 ++++++++++++++++++ .../packaging/util/Archives.java | 205 +++++++++----- .../elasticsearch/packaging/util/Cleanup.java | 39 ++- .../packaging/util/FileUtils.java | 31 ++- .../packaging/util/Installation.java | 32 +++ .../packaging/util/Platforms.java | 32 ++- .../packaging/util/ServerUtils.java | 123 +++++++++ .../elasticsearch/packaging/util/Shell.java | 52 ++-- .../packaging/tests/20_tar_package.bats | 177 ------------ 11 files changed, 675 insertions(+), 298 deletions(-) create mode 100644 qa/vagrant/src/main/java/org/elasticsearch/packaging/util/ServerUtils.java delete mode 100644 qa/vagrant/src/test/resources/packaging/tests/20_tar_package.bats diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy index 6bfe835bf39aa..90daaccd6a749 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy @@ -550,9 +550,15 @@ class VagrantTestPlugin implements Plugin { javaPackagingTest.command = 'ssh' javaPackagingTest.args = ['--command', 'sudo bash "$PACKAGING_TESTS/run-tests.sh"'] } else { + // powershell sessions run over winrm always run as administrator, whether --elevated is passed or not. however + // remote sessions have some restrictions on what they can do, such as impersonating another user (or the same user + // without administrator elevation), which we need to do for these tests. passing --elevated runs the session + // as a scheduled job locally on the vm as a true administrator to get around this limitation + // + // https://github.com/hashicorp/vagrant/blob/9c299a2a357fcf87f356bb9d56e18a037a53d138/plugins/communicators/winrm/communicator.rb#L195-L225 + // https://devops-collective-inc.gitbooks.io/secrets-of-powershell-remoting/content/manuscript/accessing-remote-computers.html javaPackagingTest.command = 'winrm' - // winrm commands run as administrator - javaPackagingTest.args = ['--command', 'powershell -File "$Env:PACKAGING_TESTS/run-tests.ps1"'] + javaPackagingTest.args = ['--elevated', '--command', 'powershell -File "$Env:PACKAGING_TESTS/run-tests.ps1"'] } TaskExecutionAdapter javaPackagingReproListener = createReproListener(project, javaPackagingTest.path) diff --git a/qa/vagrant/build.gradle b/qa/vagrant/build.gradle index 23d171f312512..704136eb4cf27 100644 --- a/qa/vagrant/build.gradle +++ b/qa/vagrant/build.gradle @@ -31,6 +31,12 @@ dependencies { compile "org.hamcrest:hamcrest-core:${versions.hamcrest}" compile "org.hamcrest:hamcrest-library:${versions.hamcrest}" + compile "org.apache.httpcomponents:httpcore:${versions.httpcore}" + compile "org.apache.httpcomponents:httpclient:${versions.httpclient}" + compile "org.apache.httpcomponents:fluent-hc:${versions.httpclient}" + compile "commons-codec:commons-codec:${versions.commonscodec}" + compile "commons-logging:commons-logging:${versions.commonslogging}" + compile project(':libs:core') // pulls in the jar built by this project and its dependencies @@ -73,3 +79,17 @@ tasks.test.enabled = false // this project doesn't get published tasks.dependencyLicenses.enabled = false tasks.dependenciesInfo.enabled = false + +tasks.thirdPartyAudit.excludes = [ + //commons-logging optional dependencies + 'org.apache.avalon.framework.logger.Logger', + 'org.apache.log.Hierarchy', + 'org.apache.log.Logger', + 'org.apache.log4j.Category', + 'org.apache.log4j.Level', + 'org.apache.log4j.Logger', + 'org.apache.log4j.Priority', + //commons-logging provided dependencies + 'javax.servlet.ServletContextEvent', + 'javax.servlet.ServletContextListener' +] diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java index ab4a11922cc21..df5e8cf995d86 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java @@ -19,6 +19,12 @@ package org.elasticsearch.packaging.test; +import org.apache.http.client.fluent.Request; +import org.elasticsearch.packaging.util.Archives; +import org.elasticsearch.packaging.util.Platforms; +import org.elasticsearch.packaging.util.ServerUtils; +import org.elasticsearch.packaging.util.Shell; +import org.elasticsearch.packaging.util.Shell.Result; import org.junit.Before; import org.junit.BeforeClass; import org.junit.FixMethodOrder; @@ -28,9 +34,33 @@ import org.elasticsearch.packaging.util.Distribution; import org.elasticsearch.packaging.util.Installation; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.stream.Stream; + +import static java.util.stream.Collectors.joining; +import static org.elasticsearch.packaging.util.Archives.ARCHIVE_OWNER; import static org.elasticsearch.packaging.util.Cleanup.cleanEverything; import static org.elasticsearch.packaging.util.Archives.installArchive; import static org.elasticsearch.packaging.util.Archives.verifyArchiveInstallation; +import static org.elasticsearch.packaging.util.FileMatcher.Fileness.File; +import static org.elasticsearch.packaging.util.FileMatcher.file; +import static org.elasticsearch.packaging.util.FileMatcher.p660; +import static org.elasticsearch.packaging.util.FileUtils.append; +import static org.elasticsearch.packaging.util.FileUtils.cp; +import static org.elasticsearch.packaging.util.FileUtils.getTempDir; +import static org.elasticsearch.packaging.util.FileUtils.mkdir; +import static org.elasticsearch.packaging.util.FileUtils.rm; +import static org.elasticsearch.packaging.util.ServerUtils.makeRequest; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.isEmptyString; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeThat; import static org.junit.Assume.assumeTrue; /** @@ -61,4 +91,226 @@ public void test10Install() { installation = installArchive(distribution()); verifyArchiveInstallation(installation, distribution()); } + + @Test + public void test20PluginsListWithNoPlugins() { + assumeThat(installation, is(notNullValue())); + + final Installation.Executables bin = installation.executables(); + final Shell sh = new Shell(); + final Result r = sh.run(bin.elasticsearchPlugin + " list"); + + assertThat(r.stdout, isEmptyString()); + } + + @Test + public void test30AbortWhenJavaMissing() { + assumeThat(installation, is(notNullValue())); + + final Installation.Executables bin = installation.executables(); + final Shell sh = new Shell(); + + Platforms.onWindows(() -> { + // on windows, removing java from PATH and removing JAVA_HOME is less involved than changing the permissions of the java + // executable. we also don't check permissions in the windows scripts anyway + final String originalPath = sh.run("$Env:PATH").stdout.trim(); + final String newPath = Arrays.stream(originalPath.split(";")) + .filter(path -> path.contains("Java") == false) + .collect(joining(";")); + + // note the lack of a $ when clearing the JAVA_HOME env variable - with a $ it deletes the java home directory + // https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/providers/environment-provider?view=powershell-6 + // + // this won't persist to another session so we don't have to reset anything + final Result runResult = sh.runIgnoreExitCode( + "$Env:PATH = '" + newPath + "'; " + + "Remove-Item Env:JAVA_HOME; " + + bin.elasticsearch + ); + + assertThat(runResult.exitCode, is(1)); + assertThat(runResult.stderr, containsString("could not find java; set JAVA_HOME or ensure java is in PATH")); + }); + + Platforms.onLinux(() -> { + final String javaPath = sh.run("which java").stdout.trim(); + + try { + sh.run("chmod -x '" + javaPath + "'"); + final Result runResult = sh.runIgnoreExitCode(bin.elasticsearch.toString()); + assertThat(runResult.exitCode, is(1)); + assertThat(runResult.stdout, containsString("could not find java; set JAVA_HOME or ensure java is in PATH")); + } finally { + sh.run("chmod +x '" + javaPath + "'"); + } + }); + } + + @Test + public void test40CreateKeystoreManually() { + assumeThat(installation, is(notNullValue())); + + final Installation.Executables bin = installation.executables(); + final Shell sh = new Shell(); + + Platforms.onLinux(() -> sh.run("sudo -u " + ARCHIVE_OWNER + " " + bin.elasticsearchKeystore + " create")); + + // this is a hack around the fact that we can't run a command in the same session as the same user but not as administrator. + // the keystore ends up being owned by the Administrators group, so we manually set it to be owned by the vagrant user here. + // from the server's perspective the permissions aren't really different, this is just to reflect what we'd expect in the tests. + // when we run these commands as a role user we won't have to do this + Platforms.onWindows(() -> sh.run( + bin.elasticsearchKeystore + " create; " + + "$account = New-Object System.Security.Principal.NTAccount 'vagrant'; " + + "$acl = Get-Acl '" + installation.config("elasticsearch.keystore") + "'; " + + "$acl.SetOwner($account); " + + "Set-Acl '" + installation.config("elasticsearch.keystore") + "' $acl" + )); + + assertThat(installation.config("elasticsearch.keystore"), file(File, ARCHIVE_OWNER, ARCHIVE_OWNER, p660)); + + Platforms.onLinux(() -> { + final Result r = sh.run("sudo -u " + ARCHIVE_OWNER + " " + bin.elasticsearchKeystore + " list"); + assertThat(r.stdout, containsString("keystore.seed")); + }); + + Platforms.onWindows(() -> { + final Result r = sh.run(bin.elasticsearchKeystore + " list"); + assertThat(r.stdout, containsString("keystore.seed")); + }); + } + + @Test + public void test50StartAndStop() throws IOException { + assumeThat(installation, is(notNullValue())); + + // cleanup from previous test + rm(installation.config("elasticsearch.keystore")); + + Archives.runElasticsearch(installation); + + final String gcLogName = Platforms.LINUX + ? "gc.log.0.current" + : "gc.log"; + assertTrue("gc logs exist", Files.exists(installation.logs.resolve(gcLogName))); + ServerUtils.runElasticsearchTests(); + + Archives.stopElasticsearch(installation); + } + + @Test + public void test60AutoCreateKeystore() { + assumeThat(installation, is(notNullValue())); + + assertThat(installation.config("elasticsearch.keystore"), file(File, ARCHIVE_OWNER, ARCHIVE_OWNER, p660)); + + final Installation.Executables bin = installation.executables(); + final Shell sh = new Shell(); + + Platforms.onLinux(() -> { + final Result result = sh.run("sudo -u " + ARCHIVE_OWNER + " " + bin.elasticsearchKeystore + " list"); + assertThat(result.stdout, containsString("keystore.seed")); + }); + + Platforms.onWindows(() -> { + final Result result = sh.run(bin.elasticsearchKeystore + " list"); + assertThat(result.stdout, containsString("keystore.seed")); + }); + } + + @Test + public void test70CustomPathConfAndJvmOptions() throws IOException { + assumeThat(installation, is(notNullValue())); + + final Path tempConf = getTempDir().resolve("esconf-alternate"); + + try { + mkdir(tempConf); + cp(installation.config("elasticsearch.yml"), tempConf.resolve("elasticsearch.yml")); + cp(installation.config("log4j2.properties"), tempConf.resolve("log4j2.properties")); + + // we have to disable Log4j from using JMX lest it will hit a security + // manager exception before we have configured logging; this will fail + // startup since we detect usages of logging before it is configured + final String jvmOptions = + "-Xms512m\n" + + "-Xmx512m\n" + + "-Dlog4j2.disable.jmx=true\n"; + append(tempConf.resolve("jvm.options"), jvmOptions); + + final Shell sh = new Shell(); + Platforms.onLinux(() -> sh.run("chown -R elasticsearch:elasticsearch " + tempConf)); + Platforms.onWindows(() -> sh.run( + "$account = New-Object System.Security.Principal.NTAccount 'vagrant'; " + + "$tempConf = Get-ChildItem '" + tempConf + "' -Recurse; " + + "$tempConf += Get-Item '" + tempConf + "'; " + + "$tempConf | ForEach-Object { " + + "$acl = Get-Acl $_.FullName; " + + "$acl.SetOwner($account); " + + "Set-Acl $_.FullName $acl " + + "}" + )); + + final Shell serverShell = new Shell(); + serverShell.getEnv().put("ES_PATH_CONF", tempConf.toString()); + serverShell.getEnv().put("ES_JAVA_OPTS", "-XX:-UseCompressedOops"); + + Archives.runElasticsearch(installation, serverShell); + + final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes")); + assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":536870912")); + assertThat(nodesResponse, containsString("\"using_compressed_ordinary_object_pointers\":\"false\"")); + + Archives.stopElasticsearch(installation); + + } finally { + rm(tempConf); + } + } + + @Test + public void test80RelativePathConf() throws IOException { + assumeThat(installation, is(notNullValue())); + + final Path temp = getTempDir().resolve("esconf-alternate"); + final Path tempConf = temp.resolve("config"); + + try { + mkdir(tempConf); + Stream.of( + "elasticsearch.yml", + "log4j2.properties", + "jvm.options" + ).forEach(file -> cp(installation.config(file), tempConf.resolve(file))); + + append(tempConf.resolve("elasticsearch.yml"), "node.name: relative"); + + final Shell sh = new Shell(); + Platforms.onLinux(() -> sh.run("chown -R elasticsearch:elasticsearch " + temp)); + Platforms.onWindows(() -> sh.run( + "$account = New-Object System.Security.Principal.NTAccount 'vagrant'; " + + "$tempConf = Get-ChildItem '" + temp + "' -Recurse; " + + "$tempConf += Get-Item '" + temp + "'; " + + "$tempConf | ForEach-Object { " + + "$acl = Get-Acl $_.FullName; " + + "$acl.SetOwner($account); " + + "Set-Acl $_.FullName $acl " + + "}" + )); + + final Shell serverShell = new Shell(temp); + serverShell.getEnv().put("ES_PATH_CONF", "config"); + Archives.runElasticsearch(installation, serverShell); + + final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes")); + assertThat(nodesResponse, containsString("\"name\":\"relative\"")); + + Archives.stopElasticsearch(installation); + + } finally { + rm(tempConf); + } + } + + } diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java index c4d3655a55398..6ffec813eb041 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java @@ -19,11 +19,14 @@ package org.elasticsearch.packaging.util; +import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.List; import java.util.stream.Stream; +import static java.util.stream.Collectors.joining; import static org.elasticsearch.packaging.util.FileMatcher.Fileness.Directory; import static org.elasticsearch.packaging.util.FileMatcher.Fileness.File; import static org.elasticsearch.packaging.util.FileMatcher.file; @@ -36,17 +39,26 @@ import static org.elasticsearch.packaging.util.FileUtils.lsGlob; import static org.elasticsearch.packaging.util.FileUtils.mv; +import static org.elasticsearch.packaging.util.FileUtils.slurp; import static org.elasticsearch.packaging.util.Platforms.isDPKG; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.isEmptyOrNullString; import static org.hamcrest.core.Is.is; import static org.hamcrest.collection.IsEmptyCollection.empty; import static org.hamcrest.collection.IsCollectionWithSize.hasSize; +import static org.hamcrest.core.IsNot.not; +import static org.junit.Assert.assertTrue; /** * Installation and verification logic for archive distributions */ public class Archives { + // in the future we'll run as a role user on Windows + public static final String ARCHIVE_OWNER = Platforms.WINDOWS + ? "vagrant" + : "elasticsearch"; + public static Installation installArchive(Distribution distribution) { return installArchive(distribution, getDefaultArchiveInstallPath(), getCurrentVersion()); } @@ -63,22 +75,20 @@ public static Installation installArchive(Distribution distribution, Path fullIn if (distribution.packaging == Distribution.Packaging.TAR) { - if (Platforms.LINUX) { - sh.bash("tar -C " + baseInstallPath + " -xzpf " + distributionFile); - } else { + Platforms.onLinux(() -> sh.run("tar -C " + baseInstallPath + " -xzpf " + distributionFile)); + + if (Platforms.WINDOWS) { throw new RuntimeException("Distribution " + distribution + " is not supported on windows"); } } else if (distribution.packaging == Distribution.Packaging.ZIP) { - if (Platforms.LINUX) { - sh.bash("unzip " + distributionFile + " -d " + baseInstallPath); - } else { - sh.powershell( - "Add-Type -AssemblyName 'System.IO.Compression.Filesystem'; " + - "[IO.Compression.ZipFile]::ExtractToDirectory('" + distributionFile + "', '" + baseInstallPath + "')" - ); - } + Platforms.onLinux(() -> sh.run("unzip " + distributionFile + " -d " + baseInstallPath)); + + Platforms.onWindows(() -> sh.run( + "Add-Type -AssemblyName 'System.IO.Compression.Filesystem'; " + + "[IO.Compression.ZipFile]::ExtractToDirectory('" + distributionFile + "', '" + baseInstallPath + "')" + )); } else { throw new RuntimeException("Distribution " + distribution + " is not a known archive type"); @@ -93,9 +103,8 @@ public static Installation installArchive(Distribution distribution, Path fullIn assertThat("only the intended installation exists", installations, hasSize(1)); assertThat("only the intended installation exists", installations.get(0), is(fullInstallPath)); - if (Platforms.LINUX) { - setupArchiveUsersLinux(fullInstallPath); - } + Platforms.onLinux(() -> setupArchiveUsersLinux(fullInstallPath)); + Platforms.onWindows(() -> setupArchiveUsersWindows(fullInstallPath)); return new Installation(fullInstallPath); } @@ -103,17 +112,17 @@ public static Installation installArchive(Distribution distribution, Path fullIn private static void setupArchiveUsersLinux(Path installPath) { final Shell sh = new Shell(); - if (sh.bashIgnoreExitCode("getent group elasticsearch").isSuccess() == false) { + if (sh.runIgnoreExitCode("getent group elasticsearch").isSuccess() == false) { if (isDPKG()) { - sh.bash("addgroup --system elasticsearch"); + sh.run("addgroup --system elasticsearch"); } else { - sh.bash("groupadd -r elasticsearch"); + sh.run("groupadd -r elasticsearch"); } } - if (sh.bashIgnoreExitCode("id elasticsearch").isSuccess() == false) { + if (sh.runIgnoreExitCode("id elasticsearch").isSuccess() == false) { if (isDPKG()) { - sh.bash("adduser " + + sh.run("adduser " + "--quiet " + "--system " + "--no-create-home " + @@ -122,7 +131,7 @@ private static void setupArchiveUsersLinux(Path installPath) { "--shell /bin/false " + "elasticsearch"); } else { - sh.bash("useradd " + + sh.run("useradd " + "--system " + "-M " + "--gid elasticsearch " + @@ -131,20 +140,29 @@ private static void setupArchiveUsersLinux(Path installPath) { "elasticsearch"); } } - sh.bash("chown -R elasticsearch:elasticsearch " + installPath); + sh.run("chown -R elasticsearch:elasticsearch " + installPath); + } + + private static void setupArchiveUsersWindows(Path installPath) { + // we want the installation to be owned as the vagrant user rather than the Administrators group + + final Shell sh = new Shell(); + sh.run( + "$account = New-Object System.Security.Principal.NTAccount 'vagrant'; " + + "$install = Get-ChildItem -Path '" + installPath + "' -Recurse; " + + "$install += Get-Item -Path '" + installPath + "'; " + + "$install | ForEach-Object { " + + "$acl = Get-Acl $_.FullName; " + + "$acl.SetOwner($account); " + + "Set-Acl $_.FullName $acl " + + "}" + ); } public static void verifyArchiveInstallation(Installation installation, Distribution distribution) { - // on Windows for now we leave the installation owned by the vagrant user that the tests run as. Since the vagrant account - // is a local administrator, the files really end up being owned by the local administrators group. In the future we'll - // install and run elasticesearch with a role user on Windows - final String owner = Platforms.WINDOWS - ? "BUILTIN\\Administrators" - : "elasticsearch"; - - verifyOssInstallation(installation, distribution, owner); + verifyOssInstallation(installation, distribution, ARCHIVE_OWNER); if (distribution.flavor == Distribution.Flavor.DEFAULT) { - verifyDefaultInstallation(installation, distribution, owner); + verifyDefaultInstallation(installation, distribution, ARCHIVE_OWNER); } } @@ -160,38 +178,38 @@ private static void verifyOssInstallation(Installation es, Distribution distribu assertThat(Files.exists(es.data), is(false)); assertThat(Files.exists(es.scripts), is(false)); - assertThat(es.home.resolve("bin"), file(Directory, owner, owner, p755)); - assertThat(es.home.resolve("lib"), file(Directory, owner, owner, p755)); - assertThat(Files.exists(es.config.resolve("elasticsearch.keystore")), is(false)); + assertThat(es.bin, file(Directory, owner, owner, p755)); + assertThat(es.lib, file(Directory, owner, owner, p755)); + assertThat(Files.exists(es.config("elasticsearch.keystore")), is(false)); Stream.of( - "bin/elasticsearch", - "bin/elasticsearch-env", - "bin/elasticsearch-keystore", - "bin/elasticsearch-plugin", - "bin/elasticsearch-translog" + "elasticsearch", + "elasticsearch-env", + "elasticsearch-keystore", + "elasticsearch-plugin", + "elasticsearch-translog" ).forEach(executable -> { - assertThat(es.home.resolve(executable), file(File, owner, owner, p755)); + assertThat(es.bin(executable), file(File, owner, owner, p755)); if (distribution.packaging == Distribution.Packaging.ZIP) { - assertThat(es.home.resolve(executable + ".bat"), file(File, owner)); + assertThat(es.bin(executable + ".bat"), file(File, owner)); } }); if (distribution.packaging == Distribution.Packaging.ZIP) { Stream.of( - "bin/elasticsearch-service.bat", - "bin/elasticsearch-service-mgr.exe", - "bin/elasticsearch-service-x64.exe" - ).forEach(executable -> assertThat(es.home.resolve(executable), file(File, owner))); + "elasticsearch-service.bat", + "elasticsearch-service-mgr.exe", + "elasticsearch-service-x64.exe" + ).forEach(executable -> assertThat(es.bin(executable), file(File, owner))); } Stream.of( "elasticsearch.yml", "jvm.options", "log4j2.properties" - ).forEach(config -> assertThat(es.config.resolve(config), file(File, owner, owner, p660))); + ).forEach(config -> assertThat(es.config(config), file(File, owner, owner, p660))); Stream.of( "NOTICE.txt", @@ -203,30 +221,30 @@ private static void verifyOssInstallation(Installation es, Distribution distribu private static void verifyDefaultInstallation(Installation es, Distribution distribution, String owner) { Stream.of( - "bin/elasticsearch-certgen", - "bin/elasticsearch-certutil", - "bin/elasticsearch-croneval", - "bin/elasticsearch-migrate", - "bin/elasticsearch-saml-metadata", - "bin/elasticsearch-setup-passwords", - "bin/elasticsearch-sql-cli", - "bin/elasticsearch-syskeygen", - "bin/elasticsearch-users", - "bin/x-pack-env", - "bin/x-pack-security-env", - "bin/x-pack-watcher-env" + "elasticsearch-certgen", + "elasticsearch-certutil", + "elasticsearch-croneval", + "elasticsearch-migrate", + "elasticsearch-saml-metadata", + "elasticsearch-setup-passwords", + "elasticsearch-sql-cli", + "elasticsearch-syskeygen", + "elasticsearch-users", + "x-pack-env", + "x-pack-security-env", + "x-pack-watcher-env" ).forEach(executable -> { - assertThat(es.home.resolve(executable), file(File, owner, owner, p755)); + assertThat(es.bin(executable), file(File, owner, owner, p755)); if (distribution.packaging == Distribution.Packaging.ZIP) { - assertThat(es.home.resolve(executable + ".bat"), file(File, owner)); + assertThat(es.bin(executable + ".bat"), file(File, owner)); } }); // at this time we only install the current version of archive distributions, but if that changes we'll need to pass // the version through here - assertThat(es.home.resolve("bin/elasticsearch-sql-cli-" + getCurrentVersion() + ".jar"), file(File, owner, owner, p755)); + assertThat(es.bin("elasticsearch-sql-cli-" + getCurrentVersion() + ".jar"), file(File, owner, owner, p755)); Stream.of( "users", @@ -234,7 +252,72 @@ private static void verifyDefaultInstallation(Installation es, Distribution dist "roles.yml", "role_mapping.yml", "log4j2.properties" - ).forEach(config -> assertThat(es.config.resolve(config), file(File, owner, owner, p660))); + ).forEach(config -> assertThat(es.config(config), file(File, owner, owner, p660))); + } + + public static void runElasticsearch(Installation installation) throws IOException { + runElasticsearch(installation, new Shell()); + } + + public static void runElasticsearch(Installation installation, Shell sh) throws IOException { + final Path pidFile = installation.home.resolve("elasticsearch.pid"); + + final Installation.Executables bin = installation.executables(); + + Platforms.onLinux(() -> { + // If jayatana is installed then we try to use it. Elasticsearch should ignore it even when we try. + // If it doesn't ignore it then Elasticsearch will fail to start because of security errors. + // This line is attempting to emulate the on login behavior of /usr/share/upstart/sessions/jayatana.conf + if (Files.exists(Paths.get("/usr/share/java/jayatanaag.jar"))) { + sh.getEnv().put("JAVA_TOOL_OPTIONS", "-javaagent:/usr/share/java/jayatanaag.jar"); + } + sh.run("sudo -E -u " + ARCHIVE_OWNER + " " + + bin.elasticsearch + " -d -p " + installation.home.resolve("elasticsearch.pid")); + }); + + Platforms.onWindows(() -> { + // this starts the server in the background. the -d flag is unsupported on windows + // these tests run as Administrator. we don't want to run the server as Administrator, so we provide the current user's + // username and password to the process which has the effect of starting it not as Administrator. + sh.run( + "$password = ConvertTo-SecureString 'vagrant' -AsPlainText -Force; " + + "$processInfo = New-Object System.Diagnostics.ProcessStartInfo; " + + "$processInfo.FileName = '" + bin.elasticsearch + "'; " + + "$processInfo.Arguments = '-p " + installation.home.resolve("elasticsearch.pid") + "'; " + + "$processInfo.Username = 'vagrant'; " + + "$processInfo.Password = $password; " + + "$processInfo.RedirectStandardOutput = $true; " + + "$processInfo.RedirectStandardError = $true; " + + sh.env.entrySet().stream() + .map(entry -> "$processInfo.Environment.Add('" + entry.getKey() + "', '" + entry.getValue() + "'); ") + .collect(joining()) + + "$processInfo.UseShellExecute = $false; " + + "$process = New-Object System.Diagnostics.Process; " + + "$process.StartInfo = $processInfo; " + + "$process.Start() | Out-Null; " + + "$process.Id;" + ); + }); + + ServerUtils.waitForElasticsearch(); + + assertTrue(Files.exists(pidFile)); + String pid = slurp(pidFile).trim(); + assertThat(pid, not(isEmptyOrNullString())); + + Platforms.onLinux(() -> sh.run("ps " + pid)); + Platforms.onWindows(() -> sh.run("Get-Process -Id " + pid)); + } + + public static void stopElasticsearch(Installation installation) { + Path pidFile = installation.home.resolve("elasticsearch.pid"); + assertTrue(Files.exists(pidFile)); + String pid = slurp(pidFile).trim(); + assertThat(pid, not(isEmptyOrNullString())); + + final Shell sh = new Shell(); + Platforms.onLinux(() -> sh.run("kill -SIGTERM " + pid)); + Platforms.onWindows(() -> sh.run("Get-Process -Id " + pid + " | Stop-Process -Force")); } } diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Cleanup.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Cleanup.java index a775a23a19490..4ff2998988c5f 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Cleanup.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Cleanup.java @@ -56,31 +56,28 @@ public static void cleanEverything() { final Shell sh = new Shell(); // kill elasticsearch processes - if (Platforms.WINDOWS) { + Platforms.onLinux(() -> { + sh.runIgnoreExitCode("pkill -u elasticsearch"); + sh.runIgnoreExitCode("ps aux | grep -i 'org.elasticsearch.bootstrap.Elasticsearch' | awk {'print $2'} | xargs kill -9"); + }); + Platforms.onWindows(() -> { // the view of processes returned by Get-Process doesn't expose command line arguments, so we use WMI here - sh.powershellIgnoreExitCode( + sh.runIgnoreExitCode( "Get-WmiObject Win32_Process | " + "Where-Object { $_.CommandLine -Match 'org.elasticsearch.bootstrap.Elasticsearch' } | " + "ForEach-Object { $_.Terminate() }" ); + }); - } else { - - sh.bashIgnoreExitCode("pkill -u elasticsearch"); - sh.bashIgnoreExitCode("ps aux | grep -i 'org.elasticsearch.bootstrap.Elasticsearch' | awk {'print $2'} | xargs kill -9"); - - } - - if (Platforms.LINUX) { - purgePackagesLinux(); - } + Platforms.onLinux(Cleanup::purgePackagesLinux); // remove elasticsearch users - if (Platforms.LINUX) { - sh.bashIgnoreExitCode("userdel elasticsearch"); - sh.bashIgnoreExitCode("groupdel elasticsearch"); - } + Platforms.onLinux(() -> { + sh.runIgnoreExitCode("userdel elasticsearch"); + sh.runIgnoreExitCode("groupdel elasticsearch"); + }); + // when we run es as a role user on windows, add the equivalent here // delete files that may still exist lsGlob(getTempDir(), "elasticsearch*").forEach(FileUtils::rm); @@ -95,7 +92,7 @@ public static void cleanEverything() { // disable elasticsearch service // todo add this for windows when adding tests for service intallation if (Platforms.LINUX && isSystemd()) { - sh.bash("systemctl unmask systemd-sysctl.service"); + sh.run("systemctl unmask systemd-sysctl.service"); } } @@ -103,19 +100,19 @@ private static void purgePackagesLinux() { final Shell sh = new Shell(); if (isRPM()) { - sh.bashIgnoreExitCode("rpm --quiet -e elasticsearch elasticsearch-oss"); + sh.runIgnoreExitCode("rpm --quiet -e elasticsearch elasticsearch-oss"); } if (isYUM()) { - sh.bashIgnoreExitCode("yum remove -y elasticsearch elasticsearch-oss"); + sh.runIgnoreExitCode("yum remove -y elasticsearch elasticsearch-oss"); } if (isDPKG()) { - sh.bashIgnoreExitCode("dpkg --purge elasticsearch elasticsearch-oss"); + sh.runIgnoreExitCode("dpkg --purge elasticsearch elasticsearch-oss"); } if (isAptGet()) { - sh.bashIgnoreExitCode("apt-get --quiet --yes purge elasticsearch elasticsearch-oss"); + sh.runIgnoreExitCode("apt-get --quiet --yes purge elasticsearch elasticsearch-oss"); } } } diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/FileUtils.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/FileUtils.java index ad826675244a0..315dc6ffee1f9 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/FileUtils.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/FileUtils.java @@ -21,11 +21,14 @@ import org.elasticsearch.core.internal.io.IOUtils; +import java.io.BufferedWriter; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.FileOwnerAttributeView; import java.nio.file.attribute.PosixFileAttributes; @@ -63,6 +66,22 @@ public static void rm(Path... paths) { } } + public static Path mkdir(Path path) { + try { + return Files.createDirectories(path); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public static Path cp(Path source, Path target) { + try { + return Files.copy(source, target); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + public static Path mv(Path source, Path target) { try { return Files.move(source, target); @@ -71,9 +90,19 @@ public static Path mv(Path source, Path target) { } } + public static void append(Path file, String text) { + try (BufferedWriter writer = Files.newBufferedWriter(file, StandardCharsets.UTF_8, + StandardOpenOption.CREATE, StandardOpenOption.APPEND)) { + + writer.write(text); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + public static String slurp(Path file) { try { - return String.join("\n", Files.readAllLines(file)); + return String.join("\n", Files.readAllLines(file, StandardCharsets.UTF_8)); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Installation.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Installation.java index d231762d06227..68da440400a36 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Installation.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Installation.java @@ -27,6 +27,8 @@ public class Installation { public final Path home; + public final Path bin; // this isn't a first-class installation feature but we include it for convenience + public final Path lib; // same public final Path config; public final Path data; public final Path logs; @@ -36,6 +38,9 @@ public class Installation { public Installation(Path home, Path config, Path data, Path logs, Path plugins, Path modules, Path scripts) { this.home = home; + this.bin = home.resolve("bin"); + this.lib = home.resolve("lib"); + this.config = config; this.data = data; this.logs = logs; @@ -55,4 +60,31 @@ public Installation(Path home) { home.resolve("scripts") ); } + + public Path bin(String executableName) { + return bin.resolve(executableName); + } + + public Path config(String configFileName) { + return config.resolve(configFileName); + } + + public Executables executables() { + return new Executables(); + } + + public class Executables { + + public final Path elasticsearch = platformExecutable("elasticsearch"); + public final Path elasticsearchPlugin = platformExecutable("elasticsearch-plugin"); + public final Path elasticsearchKeystore = platformExecutable("elasticsearch-keystore"); + public final Path elasticsearchTranslog = platformExecutable("elasticsearch-translog"); + + private Path platformExecutable(String name) { + final String platformExecutableName = Platforms.WINDOWS + ? name + ".bat" + : name; + return bin(platformExecutableName); + } + } } diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Platforms.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Platforms.java index 202c025ae8a55..5ffbc31820022 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Platforms.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Platforms.java @@ -28,41 +28,61 @@ public static boolean isDPKG() { if (WINDOWS) { return false; } - return new Shell().bashIgnoreExitCode("which dpkg").isSuccess(); + return new Shell().runIgnoreExitCode("which dpkg").isSuccess(); } public static boolean isAptGet() { if (WINDOWS) { return false; } - return new Shell().bashIgnoreExitCode("which apt-get").isSuccess(); + return new Shell().runIgnoreExitCode("which apt-get").isSuccess(); } public static boolean isRPM() { if (WINDOWS) { return false; } - return new Shell().bashIgnoreExitCode("which rpm").isSuccess(); + return new Shell().runIgnoreExitCode("which rpm").isSuccess(); } public static boolean isYUM() { if (WINDOWS) { return false; } - return new Shell().bashIgnoreExitCode("which yum").isSuccess(); + return new Shell().runIgnoreExitCode("which yum").isSuccess(); } public static boolean isSystemd() { if (WINDOWS) { return false; } - return new Shell().bashIgnoreExitCode("which systemctl").isSuccess(); + return new Shell().runIgnoreExitCode("which systemctl").isSuccess(); } public static boolean isSysVInit() { if (WINDOWS) { return false; } - return new Shell().bashIgnoreExitCode("which service").isSuccess(); + return new Shell().runIgnoreExitCode("which service").isSuccess(); + } + + public static void onWindows(PlatformAction action) { + if (WINDOWS) { + action.run(); + } + } + + public static void onLinux(PlatformAction action) { + if (LINUX) { + action.run(); + } + } + + /** + * Essentially a Runnable, but we make the distinction so it's more clear that these are synchronous + */ + @FunctionalInterface + public interface PlatformAction { + void run(); } } diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/ServerUtils.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/ServerUtils.java new file mode 100644 index 0000000000000..ff006a34e6892 --- /dev/null +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/ServerUtils.java @@ -0,0 +1,123 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.packaging.util; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.http.HttpResponse; +import org.apache.http.client.fluent.Request; +import org.apache.http.conn.HttpHostConnectException; +import org.apache.http.entity.ContentType; +import org.apache.http.util.EntityUtils; + +import java.io.IOException; +import java.util.Objects; +import java.util.concurrent.TimeUnit; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; + +public class ServerUtils { + + private static final Log LOG = LogFactory.getLog(ServerUtils.class); + + private static final long waitTime = TimeUnit.SECONDS.toMillis(60); + private static final long timeoutLength = TimeUnit.SECONDS.toMillis(10); + + public static void waitForElasticsearch() throws IOException { + waitForElasticsearch("green", null); + } + + public static void waitForElasticsearch(String status, String index) throws IOException { + + Objects.requireNonNull(status); + + // we loop here rather than letting httpclient handle retries so we can measure the entire waiting time + final long startTime = System.currentTimeMillis(); + long timeElapsed = 0; + boolean started = false; + while (started == false && timeElapsed < waitTime) { + try { + + final HttpResponse response = Request.Get("http://localhost:9200/_cluster/health") + .connectTimeout((int) timeoutLength) + .socketTimeout((int) timeoutLength) + .execute() + .returnResponse(); + + if (response.getStatusLine().getStatusCode() >= 300) { + final String statusLine = response.getStatusLine().toString(); + final String body = EntityUtils.toString(response.getEntity()); + throw new RuntimeException("Connecting to elasticsearch cluster health API failed:\n" + statusLine+ "\n" + body); + } + + started = true; + + } catch (HttpHostConnectException e) { + // we want to retry if the connection is refused + LOG.info("Got connection refused when waiting for cluster health", e); + } + + timeElapsed = System.currentTimeMillis() - startTime; + } + + if (started == false) { + throw new RuntimeException("Elasticsearch did not start"); + } + + final String url; + if (index == null) { + url = "http://localhost:9200/_cluster/health?wait_for_status=" + status + "&timeout=60s&pretty"; + } else { + url = "http://localhost:9200/_cluster/health/" + index + "?wait_for_status=" + status + "&timeout=60s&pretty"; + + } + + final String body = makeRequest(Request.Get(url)); + assertThat("cluster health response must contain desired status", body, containsString(status)); + } + + public static void runElasticsearchTests() throws IOException { + makeRequest( + Request.Post("http://localhost:9200/library/book/1?refresh=true&pretty") + .bodyString("{ \"title\": \"Book #1\", \"pages\": 123 }", ContentType.APPLICATION_JSON)); + + makeRequest( + Request.Post("http://localhost:9200/library/book/2?refresh=true&pretty") + .bodyString("{ \"title\": \"Book #2\", \"pages\": 456 }", ContentType.APPLICATION_JSON)); + + String count = makeRequest(Request.Get("http://localhost:9200/_count?pretty")); + assertThat(count, containsString("\"count\" : 2")); + + makeRequest(Request.Delete("http://localhost:9200/_all")); + } + + public static String makeRequest(Request request) throws IOException { + final HttpResponse response = request.execute().returnResponse(); + final String body = EntityUtils.toString(response.getEntity()); + + if (response.getStatusLine().getStatusCode() >= 300) { + throw new RuntimeException("Request failed:\n" + response.getStatusLine().toString() + "\n" + body); + } + + return body; + + } +} diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Shell.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Shell.java index 9a908e2d680eb..5853bc2daa148 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Shell.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Shell.java @@ -58,58 +58,50 @@ public Shell(Map env, Path workingDirectory) { this.workingDirectory = workingDirectory; } - /** - * Runs a script in a bash shell, throwing an exception if its exit code is nonzero - */ - public Result bash(String script) { - return run(bashCommand(script)); + public Map getEnv() { + return env; } /** - * Runs a script in a bash shell + * Run the provided string as a shell script. On Linux the {@code bash -c [script]} syntax will be used, and on Windows + * the {@code powershell.exe -Command [script]} syntax will be used. Throws an exception if the exit code of the script is nonzero */ - public Result bashIgnoreExitCode(String script) { - return runIgnoreExitCode(bashCommand(script)); - } - - private static String[] bashCommand(String script) { - return Stream.concat(Stream.of("bash", "-c"), Stream.of(script)).toArray(String[]::new); + public Result run(String script) { + return runScript(getScriptCommand(script)); } /** - * Runs a script in a powershell shell, throwing an exception if its exit code is nonzero + * Same as {@link #run(String)}, but does not throw an exception if the exit code of the script is nonzero */ - public Result powershell(String script) { - return run(powershellCommand(script)); + public Result runIgnoreExitCode(String script) { + return runScriptIgnoreExitCode(getScriptCommand(script)); } - /** - * Runs a script in a powershell shell - */ - public Result powershellIgnoreExitCode(String script) { - return runIgnoreExitCode(powershellCommand(script)); + private String[] getScriptCommand(String script) { + if (Platforms.WINDOWS) { + return powershellCommand(script); + } else { + return bashCommand(script); + } + } + + private static String[] bashCommand(String script) { + return Stream.concat(Stream.of("bash", "-c"), Stream.of(script)).toArray(String[]::new); } private static String[] powershellCommand(String script) { return Stream.concat(Stream.of("powershell.exe", "-Command"), Stream.of(script)).toArray(String[]::new); } - /** - * Runs an executable file, passing all elements of {@code command} after the first as arguments. Throws an exception if the process' - * exit code is nonzero - */ - private Result run(String[] command) { - Result result = runIgnoreExitCode(command); + private Result runScript(String[] command) { + Result result = runScriptIgnoreExitCode(command); if (result.isSuccess() == false) { throw new RuntimeException("Command was not successful: [" + String.join(" ", command) + "] result: " + result.toString()); } return result; } - /** - * Runs an executable file, passing all elements of {@code command} after the first as arguments - */ - private Result runIgnoreExitCode(String[] command) { + private Result runScriptIgnoreExitCode(String[] command) { ProcessBuilder builder = new ProcessBuilder(); builder.command(command); diff --git a/qa/vagrant/src/test/resources/packaging/tests/20_tar_package.bats b/qa/vagrant/src/test/resources/packaging/tests/20_tar_package.bats deleted file mode 100644 index 1a3704c33172f..0000000000000 --- a/qa/vagrant/src/test/resources/packaging/tests/20_tar_package.bats +++ /dev/null @@ -1,177 +0,0 @@ -#!/usr/bin/env bats - -# This file is used to test the tar gz package. - -# WARNING: This testing file must be executed as root and can -# dramatically change your system. It should only be executed -# in a throw-away VM like those made by the Vagrantfile at -# the root of the Elasticsearch source code. This should -# cause the script to fail if it is executed any other way: -[ -f /etc/is_vagrant_vm ] || { - >&2 echo "must be run on a vagrant VM" - exit 1 -} - -# The test case can be executed with the Bash Automated -# Testing System tool available at https://github.com/sstephenson/bats -# Thanks to Sam Stephenson! - -# Licensed to Elasticsearch under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Load test utilities -load $BATS_UTILS/utils.bash -load $BATS_UTILS/tar.bash -load $BATS_UTILS/plugins.bash - -setup() { - skip_not_tar_gz - export ESHOME=/tmp/elasticsearch - export_elasticsearch_paths -} - -################################## -# Install TAR GZ package -################################## -@test "[TAR] tar command is available" { - # Cleans everything for the 1st execution - clean_before_test - run tar --version - [ "$status" -eq 0 ] -} - -@test "[TAR] archive is available" { - local version=$(cat version) - count=$(find . -type f -name "${PACKAGE_NAME}-${version}.tar.gz" | wc -l) - [ "$count" -eq 1 ] -} - -@test "[TAR] archive is not installed" { - count=$(find /tmp -type d -name 'elasticsearch*' | wc -l) - [ "$count" -eq 0 ] -} - -@test "[TAR] install archive" { - # Install the archive - install_archive - - count=$(find /tmp -type d -name 'elasticsearch*' | wc -l) - [ "$count" -eq 1 ] - - # Its simpler to check that the install was correct in this test rather - # than in another test because install_archive sets a number of path - # variables that verify_archive_installation reads. To separate this into - # another test you'd have to recreate the variables. - verify_archive_installation -} - -@test "[TAR] verify elasticsearch-plugin list runs without any plugins installed" { - # previously this would fail because the archive installations did - # not create an empty plugins directory - local plugins_list=`$ESHOME/bin/elasticsearch-plugin list` - [[ -z $plugins_list ]] -} - -@test "[TAR] elasticsearch fails if java executable is not found" { - local JAVA=$(which java) - - sudo chmod -x $JAVA - run "$ESHOME/bin/elasticsearch" - sudo chmod +x $JAVA - - [ "$status" -eq 1 ] - local expected="could not find java; set JAVA_HOME or ensure java is in PATH" - [[ "$output" == *"$expected"* ]] || { - echo "Expected error message [$expected] but found: $output" - false - } -} - -@test "[TAR] test creating elasticearch.keystore" { - sudo -E -u elasticsearch "$ESHOME/bin/elasticsearch-keystore" create - assert_file "$ESCONFIG/elasticsearch.keystore" f elasticsearch elasticsearch 660 - sudo -E -u elasticsearch "$ESHOME/bin/elasticsearch-keystore" list | grep "keystore.seed" - # cleanup for the next test - rm -rf "$ESCONFIG/elasticsearch.keystore" -} - -################################## -# Check that Elasticsearch is working -################################## -@test "[TAR] test elasticsearch" { - start_elasticsearch_service - run_elasticsearch_tests - stop_elasticsearch_service -} - -@test "[TAR] test auto-creating elasticearch.keystore" { - # a keystore should automatically be created after the service is started - assert_file "$ESCONFIG/elasticsearch.keystore" f elasticsearch elasticsearch 660 - # the keystore should be seeded - sudo -E -u elasticsearch "$ESHOME/bin/elasticsearch-keystore" list | grep "keystore.seed" -} - -@test "[TAR] start Elasticsearch with custom JVM options" { - local es_java_opts=$ES_JAVA_OPTS - local es_path_conf=$ES_PATH_CONF - local temp=`mktemp -d` - cp "$ESCONFIG"/elasticsearch.yml "$temp" - cp "$ESCONFIG"/log4j2.properties "$temp" - touch "$temp/jvm.options" - chown -R elasticsearch:elasticsearch "$temp" - echo "-Xms512m" >> "$temp/jvm.options" - echo "-Xmx512m" >> "$temp/jvm.options" - # we have to disable Log4j from using JMX lest it will hit a security - # manager exception before we have configured logging; this will fail - # startup since we detect usages of logging before it is configured - echo "-Dlog4j2.disable.jmx=true" >> "$temp/jvm.options" - export ES_PATH_CONF="$temp" - export ES_JAVA_OPTS="-XX:-UseCompressedOops" - start_elasticsearch_service - curl -s -XGET localhost:9200/_nodes | fgrep '"heap_init_in_bytes":536870912' - curl -s -XGET localhost:9200/_nodes | fgrep '"using_compressed_ordinary_object_pointers":"false"' - stop_elasticsearch_service - export ES_PATH_CONF=$es_path_conf - export ES_JAVA_OPTS=$es_java_opts -} - -@test "[TAR] GC logs exist" { - start_elasticsearch_service - assert_file_exist $ESHOME/logs/gc.log.0.current - stop_elasticsearch_service -} - -@test "[TAR] relative ES_PATH_CONF" { - local es_path_conf=$ES_PATH_CONF - local temp=`mktemp -d` - mkdir "$temp"/config - cp "$ESCONFIG"/elasticsearch.yml "$temp"/config - cp "$ESCONFIG"/log4j2.properties "$temp"/config - cp "$ESCONFIG/jvm.options" "$temp/config" - chown -R elasticsearch:elasticsearch "$temp" - echo "node.name: relative" >> "$temp"/config/elasticsearch.yml - cd "$temp" - export ES_PATH_CONF=config - start_elasticsearch_service - curl -s -XGET localhost:9200/_nodes | fgrep '"name":"relative"' - stop_elasticsearch_service - export ES_PATH_CONF=$es_path_conf -} - -@test "[TAR] remove tar" { - rm -rf "/tmp/elasticsearch" -} From 27c74d84f569658d0105a61448c64f3168250393 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Tue, 10 Jul 2018 10:52:03 +0200 Subject: [PATCH 19/50] rolling upgrade should use a replica to prevent relocations while running a scroll A scroll holds a reference to the shard store. If the cluster is moving shards around that reference can prevent a shard from relocating back to node it used to be on, causing test failures. Closes #31827 --- .../resources/rest-api-spec/test/mixed_cluster/10_basic.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml index 770e694524fe3..4596e95bba1d3 100644 --- a/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml +++ b/x-pack/qa/rolling-upgrade/src/test/resources/rest-api-spec/test/mixed_cluster/10_basic.yml @@ -6,7 +6,10 @@ wait_for_active_shards: all body: settings: - number_of_replicas: 0 +# we use 1 replica to make sure we don't have shards relocating. Relocating a shard with +# a scroll on it prevents shards from moving back into a where a scroll is running (it holds the shard lock) +# see https://github.com/elastic/elasticsearch/issues/31827 + number_of_replicas: 1 index.routing.allocation.include.upgraded: true - do: From 030f4062e9e248067e947111dc1aabe39a723124 Mon Sep 17 00:00:00 2001 From: Nhat Nguyen Date: Tue, 10 Jul 2018 17:44:26 -0400 Subject: [PATCH 20/50] Increase logging level for testStressMaybeFlush Relates #31629 --- .../elasticsearch/index/shard/IndexShardIT.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java index e8f396800458b..57c430e66ff7d 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardIT.java @@ -70,6 +70,7 @@ import org.elasticsearch.test.ESSingleNodeTestCase; import org.elasticsearch.test.IndexSettingsModule; import org.elasticsearch.test.InternalSettingsPlugin; +import org.elasticsearch.test.junit.annotations.TestLogging; import java.io.IOException; import java.io.UncheckedIOException; @@ -81,6 +82,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Locale; import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.atomic.AtomicBoolean; @@ -398,6 +400,7 @@ public void testMaybeRollTranslogGeneration() throws Exception { } } + @TestLogging("_root:DEBUG,org.elasticsearch.index.shard:TRACE,org.elasticsearch.index.engine:TRACE") public void testStressMaybeFlushOrRollTranslogGeneration() throws Exception { createIndex("test"); ensureGreen(); @@ -440,13 +443,14 @@ public void testStressMaybeFlushOrRollTranslogGeneration() throws Exception { barrier.await(); final CheckedRunnable check; if (flush) { - final FlushStats flushStats = shard.flushStats(); - final long total = flushStats.getTotal(); - final long periodic = flushStats.getPeriodic(); + final FlushStats initialStats = shard.flushStats(); client().prepareIndex("test", "test", "1").setSource("{}", XContentType.JSON).get(); check = () -> { - assertThat(shard.flushStats().getTotal(), equalTo(total + 1)); - assertThat(shard.flushStats().getPeriodic(), equalTo(periodic + 1)); + final FlushStats currentStats = shard.flushStats(); + String msg = String.format(Locale.ROOT, "flush stats: total=[%d vs %d], periodic=[%d vs %d]", + initialStats.getTotal(), currentStats.getTotal(), initialStats.getPeriodic(), currentStats.getPeriodic()); + assertThat(msg, currentStats.getPeriodic(), equalTo(initialStats.getPeriodic() + 1)); + assertThat(msg, currentStats.getTotal(), equalTo(initialStats.getTotal() + 1)); }; } else { final long generation = getTranslog(shard).currentFileGeneration(); From 119090f12b7def5fa540fda69e7ce5b312f7ec70 Mon Sep 17 00:00:00 2001 From: David Kyle Date: Wed, 11 Jul 2018 10:17:44 +0100 Subject: [PATCH 21/50] [ML] Mute test failing due to Java 11 date time format parsing bug (#31899) --- .../xpack/core/ml/job/config/DataDescription.java | 3 ++- .../time/DateTimeFormatterTimestampConverter.java | 6 +++--- .../core/ml/job/config/DataDescriptionTests.java | 14 ++++++++++++-- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/DataDescription.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/DataDescription.java index 9ff578be50b85..6e9652bdfa263 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/DataDescription.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/DataDescription.java @@ -353,7 +353,8 @@ public void setTimeFormat(String format) { try { DateTimeFormatterTimestampConverter.ofPattern(format, ZoneOffset.UTC); } catch (IllegalArgumentException e) { - throw ExceptionsHelper.badRequestException(Messages.getMessage(Messages.JOB_CONFIG_INVALID_TIMEFORMAT, format)); + throw ExceptionsHelper.badRequestException( + Messages.getMessage(Messages.JOB_CONFIG_INVALID_TIMEFORMAT, format), e.getCause()); } } timeFormat = format; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/time/DateTimeFormatterTimestampConverter.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/time/DateTimeFormatterTimestampConverter.java index 556c2f37b485d..0efb5feb38b91 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/time/DateTimeFormatterTimestampConverter.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/utils/time/DateTimeFormatterTimestampConverter.java @@ -54,9 +54,9 @@ public static TimestampConverter ofPattern(String pattern, ZoneId defaultTimezon .parseDefaulting(ChronoField.YEAR_OF_ERA, LocalDate.now(defaultTimezone).getYear()) .toFormatter(); - String now = formatter.format(ZonedDateTime.ofInstant(Instant.ofEpochSecond(0), ZoneOffset.UTC)); + String formattedTime = formatter.format(ZonedDateTime.ofInstant(Instant.ofEpochSecond(0), ZoneOffset.UTC)); try { - TemporalAccessor parsed = formatter.parse(now); + TemporalAccessor parsed = formatter.parse(formattedTime); boolean hasTimeZone = parsed.isSupported(ChronoField.INSTANT_SECONDS); if (hasTimeZone) { Instant.from(parsed); @@ -67,7 +67,7 @@ public static TimestampConverter ofPattern(String pattern, ZoneId defaultTimezon return new DateTimeFormatterTimestampConverter(formatter, hasTimeZone, defaultTimezone); } catch (DateTimeException e) { - throw new IllegalArgumentException("Timestamp cannot be derived from pattern: " + pattern); + throw new IllegalArgumentException("Timestamp cannot be derived from pattern: " + pattern, e); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DataDescriptionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DataDescriptionTests.java index 3ca4bac47cb29..bb7c329cf4508 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DataDescriptionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/DataDescriptionTests.java @@ -17,6 +17,8 @@ import org.elasticsearch.xpack.core.ml.job.config.DataDescription.DataFormat; import org.elasticsearch.xpack.core.ml.job.messages.Messages; +import java.time.DateTimeException; + import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; @@ -51,8 +53,12 @@ public void testVerify_GivenValidFormat() { description.setTimeFormat("epoch"); description.setTimeFormat("epoch_ms"); description.setTimeFormat("yyyy-MM-dd HH"); - String goodFormat = "yyyy.MM.dd G 'at' HH:mm:ss z"; - description.setTimeFormat(goodFormat); + } + + @AwaitsFix(bugUrl = "https://bugs.java.com/bugdatabase/view_bug.do?bug_id=JDK-8206980") + public void testVerify_GivenValidFormat_Java11Bug() { + DataDescription.Builder description = new DataDescription.Builder(); + description.setTimeFormat("yyyy.MM.dd G 'at' HH:mm:ss z"); } public void testVerify_GivenInValidFormat() { @@ -68,6 +74,10 @@ public void testVerify_GivenInValidFormat() { e = expectThrows(ElasticsearchException.class, () -> description.setTimeFormat("y-M-dd")); assertEquals(Messages.getMessage(Messages.JOB_CONFIG_INVALID_TIMEFORMAT, "y-M-dd"), e.getMessage()); expectThrows(ElasticsearchException.class, () -> description.setTimeFormat("YYY-mm-UU hh:mm:ssY")); + + Throwable cause = e.getCause(); + assertNotNull(cause); + assertThat(cause, instanceOf(DateTimeException.class)); } public void testTransform_GivenDelimitedAndEpoch() { From 8b174c4b560e812a7d8703ce62a124498c057ee5 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 10 Jul 2018 17:59:15 -0400 Subject: [PATCH 22/50] HLREST: Bundle the x-pack protocol project (#31904) The `:x-pack:protocol` project is an implementation detail shared by the xpack projects and the high level rest client and really doesn't deserve its own maven coordinants and published javadoc. This change bundles `:x-pack:protocol` into the high level rest client. Relates to #29827 --- client/rest-high-level/build.gradle | 123 ++++++++++++++++++++--- qa/ccs-unavailable-clusters/build.gradle | 4 +- 2 files changed, 113 insertions(+), 14 deletions(-) diff --git a/client/rest-high-level/build.gradle b/client/rest-high-level/build.gradle index 05b1cb25ed5d0..451452759f507 100644 --- a/client/rest-high-level/build.gradle +++ b/client/rest-high-level/build.gradle @@ -1,5 +1,3 @@ -import org.elasticsearch.gradle.precommit.PrecommitTasks - /* * Licensed to Elasticsearch under one or more contributor * license agreements. See the NOTICE file distributed with @@ -18,30 +16,86 @@ import org.elasticsearch.gradle.precommit.PrecommitTasks * specific language governing permissions and limitations * under the License. */ + +import org.elasticsearch.gradle.precommit.PrecommitTasks +import org.gradle.api.XmlProvider +import org.gradle.api.publish.maven.MavenPublication + +buildscript { + repositories { + maven { + url 'https://plugins.gradle.org/m2/' + } + } + dependencies { + classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.4' + } +} + apply plugin: 'elasticsearch.build' apply plugin: 'elasticsearch.rest-test' apply plugin: 'nebula.maven-base-publish' apply plugin: 'nebula.maven-scm' +apply plugin: 'com.github.johnrengelman.shadow' group = 'org.elasticsearch.client' archivesBaseName = 'elasticsearch-rest-high-level-client' publishing { - publications { - nebula { - artifactId = archivesBaseName + publications { + nebula(MavenPublication) { + artifact shadowJar + artifactId = archivesBaseName + /* + * Configure the pom to include the "shadow" as compile dependencies + * because that is how we're using them but remove all other dependencies + * because they've been shaded into the jar. + */ + pom.withXml { XmlProvider xml -> + Node root = xml.asNode() + root.remove(root.dependencies) + Node dependenciesNode = root.appendNode('dependencies') + project.configurations.shadow.allDependencies.each { + if (false == it instanceof SelfResolvingDependency) { + Node dependencyNode = dependenciesNode.appendNode('dependency') + dependencyNode.appendNode('groupId', it.group) + dependencyNode.appendNode('artifactId', it.name) + dependencyNode.appendNode('version', it.version) + dependencyNode.appendNode('scope', 'compile') + } } + } } + } +} + +/* + * We need somewhere to configure dependencies that we don't wish to shade + * into the high level REST client. The shadow plugin creates a "shadow" + * configuration which is *almost* exactly that. It is never bundled into + * the shaded jar but is used for main source compilation. Unfortunately, + * by default it is not used for *test* source compilation and isn't used + * in tests at all. This change makes it available for test compilation. + * A change below makes it available for testing. + */ +sourceSets { + test { + compileClasspath += configurations.shadow + } } dependencies { - compile "org.elasticsearch:elasticsearch:${version}" - compile "org.elasticsearch.client:elasticsearch-rest-client:${version}" - compile "org.elasticsearch.plugin:parent-join-client:${version}" - compile "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}" - compile "org.elasticsearch.plugin:rank-eval-client:${version}" - compile "org.elasticsearch.plugin:lang-mustache-client:${version}" - compile project(':x-pack:protocol') // TODO bundle into the jar + /* + * Everything in the "shadow" configuration is *not* copied into the + * shadowJar. + */ + shadow "org.elasticsearch:elasticsearch:${version}" + shadow "org.elasticsearch.client:elasticsearch-rest-client:${version}" + shadow "org.elasticsearch.plugin:parent-join-client:${version}" + shadow "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}" + shadow "org.elasticsearch.plugin:rank-eval-client:${version}" + shadow "org.elasticsearch.plugin:lang-mustache-client:${version}" + compile project(':x-pack:protocol') testCompile "org.elasticsearch.client:test:${version}" testCompile "org.elasticsearch.test:framework:${version}" @@ -64,3 +118,48 @@ forbiddenApisMain { signaturesURLs += [PrecommitTasks.getResource('/forbidden/http-signatures.txt')] signaturesURLs += [file('src/main/resources/forbidden/rest-high-level-signatures.txt').toURI().toURL()] } + +shadowJar { + classifier = null + mergeServiceFiles() +} + +// We don't need normal jar, we use shadow jar instead +jar.enabled = false +assemble.dependsOn shadowJar + +javadoc { + /* + * Bundle all of the javadoc from all of the shaded projects into this one + * so we don't *have* to publish javadoc for all of the "client" jars. + */ + configurations.compile.dependencies.all { Dependency dep -> + Project p = dependencyToProject(dep) + if (p != null) { + evaluationDependsOn(p.path) + source += p.sourceSets.main.allJava + } + } +} + +/* + * Use the jar for testing so we have tests of the bundled jar. + * Use the "shadow" configuration for testing because we need things + * in it. + */ +test { + classpath -= compileJava.outputs.files + classpath -= configurations.compile + classpath -= configurations.runtime + classpath += configurations.shadow + classpath += shadowJar.outputs.files + dependsOn shadowJar +} +integTestRunner { + classpath -= compileJava.outputs.files + classpath -= configurations.compile + classpath -= configurations.runtime + classpath += configurations.shadow + classpath += shadowJar.outputs.files + dependsOn shadowJar +} diff --git a/qa/ccs-unavailable-clusters/build.gradle b/qa/ccs-unavailable-clusters/build.gradle index 86d0cb64f65a2..d9de422bb43e1 100644 --- a/qa/ccs-unavailable-clusters/build.gradle +++ b/qa/ccs-unavailable-clusters/build.gradle @@ -21,5 +21,5 @@ apply plugin: 'elasticsearch.rest-test' apply plugin: 'elasticsearch.test-with-dependencies' dependencies { - testCompile project(path: ':client:rest-high-level', configuration: 'runtime') -} \ No newline at end of file + testCompile project(path: ':client:rest-high-level', configuration: 'shadow') +} From 1c60a322677f0a766f1ff6056e2a1be44da3e59c Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 11 Jul 2018 09:18:04 -0400 Subject: [PATCH 23/50] Switch high level rest tests to new style requests (#31937) In #29623 we added `Request` object flavored requests to the low level REST client and in #30315 we deprecated the old `performRequest`s. This changes all calls in the `client/rest-high-level` project to use the new versions. --- .../elasticsearch/client/IndicesClientIT.java | 16 +- .../elasticsearch/client/PingAndInfoIT.java | 2 +- .../org/elasticsearch/client/RankEvalIT.java | 39 ++- .../org/elasticsearch/client/SearchIT.java | 268 ++++++++++-------- .../documentation/CRUDDocumentationIT.java | 43 ++- 5 files changed, 194 insertions(+), 174 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java index 5a355b1334130..fadc9ade58705 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java @@ -631,7 +631,7 @@ public void testOpenExistingIndex() throws IOException { createIndex(index, Settings.EMPTY); closeIndex(index); ResponseException exception = expectThrows(ResponseException.class, - () -> client().performRequest(HttpGet.METHOD_NAME, index + "/_search")); + () -> client().performRequest(new Request(HttpGet.METHOD_NAME, index + "/_search"))); assertThat(exception.getResponse().getStatusLine().getStatusCode(), equalTo(RestStatus.BAD_REQUEST.getStatus())); assertThat(exception.getMessage().contains(index), equalTo(true)); @@ -642,7 +642,7 @@ public void testOpenExistingIndex() throws IOException { assertTrue(openIndexResponse.isAcknowledged()); assertTrue(openIndexResponse.isShardsAcknowledged()); - Response response = client().performRequest(HttpGet.METHOD_NAME, index + "/_search"); + Response response = client().performRequest(new Request(HttpGet.METHOD_NAME, index + "/_search")); assertThat(response.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus())); } @@ -674,7 +674,7 @@ public void testOpenNonExistentIndex() throws IOException { public void testCloseExistingIndex() throws IOException { String index = "index"; createIndex(index, Settings.EMPTY); - Response response = client().performRequest(HttpGet.METHOD_NAME, index + "/_search"); + Response response = client().performRequest(new Request(HttpGet.METHOD_NAME, index + "/_search")); assertThat(response.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus())); CloseIndexRequest closeIndexRequest = new CloseIndexRequest(index); @@ -684,7 +684,7 @@ public void testCloseExistingIndex() throws IOException { assertTrue(closeIndexResponse.isAcknowledged()); ResponseException exception = expectThrows(ResponseException.class, - () -> client().performRequest(HttpGet.METHOD_NAME, index + "/_search")); + () -> client().performRequest(new Request(HttpGet.METHOD_NAME, index + "/_search"))); assertThat(exception.getResponse().getStatusLine().getStatusCode(), equalTo(RestStatus.BAD_REQUEST.getStatus())); assertThat(exception.getMessage().contains(index), equalTo(true)); } @@ -851,7 +851,7 @@ public void testExistsAlias() throws IOException { highLevelClient().indices()::existsAlias, highLevelClient().indices()::existsAliasAsync)); createIndex("index", Settings.EMPTY); - client().performRequest(HttpPut.METHOD_NAME, "/index/_alias/alias"); + client().performRequest(new Request(HttpPut.METHOD_NAME, "/index/_alias/alias")); assertTrue(execute(getAliasesRequest, highLevelClient().indices()::existsAlias, highLevelClient().indices()::existsAliasAsync, highLevelClient().indices()::existsAlias, highLevelClient().indices()::existsAliasAsync)); @@ -972,10 +972,10 @@ public void testRollover() throws IOException { public void testGetAlias() throws IOException { { createIndex("index1", Settings.EMPTY); - client().performRequest(HttpPut.METHOD_NAME, "/index1/_alias/alias1"); + client().performRequest(new Request(HttpPut.METHOD_NAME, "/index1/_alias/alias1")); createIndex("index2", Settings.EMPTY); - client().performRequest(HttpPut.METHOD_NAME, "/index2/_alias/alias2"); + client().performRequest(new Request(HttpPut.METHOD_NAME, "/index2/_alias/alias2")); createIndex("index3", Settings.EMPTY); } @@ -1111,7 +1111,7 @@ public void testGetAliasesNonExistentIndexOrAlias() throws IOException { assertThat(getAliasesResponse.getError(), equalTo("alias [" + alias + "] missing")); } createIndex(index, Settings.EMPTY); - client().performRequest(HttpPut.METHOD_NAME, index + "/_alias/" + alias); + client().performRequest(new Request(HttpPut.METHOD_NAME, index + "/_alias/" + alias)); { GetAliasesRequest getAliasesRequest = new GetAliasesRequest().indices(index, "non_existent_index"); GetAliasesResponse getAliasesResponse = execute(getAliasesRequest, highLevelClient().indices()::getAlias, diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/PingAndInfoIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/PingAndInfoIT.java index 1a50187f5df5d..b45f52f9e441c 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/PingAndInfoIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/PingAndInfoIT.java @@ -39,7 +39,7 @@ public void testPing() throws IOException { public void testInfo() throws IOException { MainResponse info = highLevelClient().info(RequestOptions.DEFAULT); // compare with what the low level client outputs - Map infoAsMap = entityAsMap(adminClient().performRequest(HttpGet.METHOD_NAME, "/")); + Map infoAsMap = entityAsMap(adminClient().performRequest(new Request(HttpGet.METHOD_NAME, "/"))); assertEquals(infoAsMap.get("cluster_name"), info.getClusterName().value()); assertEquals(infoAsMap.get("cluster_uuid"), info.getClusterUuid()); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java index 1e12f3f5e62f6..4a39a0e527acd 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java @@ -19,8 +19,6 @@ package org.elasticsearch.client; -import org.apache.http.entity.ContentType; -import org.apache.http.entity.StringEntity; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.index.query.MatchAllQueryBuilder; @@ -37,7 +35,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -49,19 +46,17 @@ public class RankEvalIT extends ESRestHighLevelClientTestCase { @Before public void indexDocuments() throws IOException { - StringEntity doc = new StringEntity("{\"text\":\"berlin\"}", ContentType.APPLICATION_JSON); - client().performRequest("PUT", "/index/doc/1", Collections.emptyMap(), doc); - doc = new StringEntity("{\"text\":\"amsterdam\"}", ContentType.APPLICATION_JSON); - client().performRequest("PUT", "/index/doc/2", Collections.emptyMap(), doc); - client().performRequest("PUT", "/index/doc/3", Collections.emptyMap(), doc); - client().performRequest("PUT", "/index/doc/4", Collections.emptyMap(), doc); - client().performRequest("PUT", "/index/doc/5", Collections.emptyMap(), doc); - client().performRequest("PUT", "/index/doc/6", Collections.emptyMap(), doc); - client().performRequest("POST", "/index/_refresh"); - - // add another index to test basic multi index support - client().performRequest("PUT", "/index2/doc/7", Collections.emptyMap(), doc); - client().performRequest("POST", "/index2/_refresh"); + Request berlin = new Request("PUT", "/index/doc/berlin"); + berlin.setJsonEntity("{\"text\":\"berlin\"}"); + client().performRequest(berlin); + for (int i = 0; i < 6; i++) { + // add another index to test basic multi index support + String index = i == 0 ? "index2" : "index"; + Request amsterdam = new Request("PUT", "/" + index + "/doc/amsterdam" + i); + amsterdam.setJsonEntity("{\"text\":\"amsterdam\"}"); + client().performRequest(amsterdam); + } + client().performRequest(new Request("POST", "/_refresh")); } /** @@ -71,10 +66,10 @@ public void indexDocuments() throws IOException { public void testRankEvalRequest() throws IOException { SearchSourceBuilder testQuery = new SearchSourceBuilder(); testQuery.query(new MatchAllQueryBuilder()); - List amsterdamRatedDocs = createRelevant("index" , "2", "3", "4", "5"); - amsterdamRatedDocs.addAll(createRelevant("index2", "7")); + List amsterdamRatedDocs = createRelevant("index" , "amsterdam1", "amsterdam2", "amsterdam3", "amsterdam4"); + amsterdamRatedDocs.addAll(createRelevant("index2", "amsterdam0")); RatedRequest amsterdamRequest = new RatedRequest("amsterdam_query", amsterdamRatedDocs, testQuery); - RatedRequest berlinRequest = new RatedRequest("berlin_query", createRelevant("index", "1"), testQuery); + RatedRequest berlinRequest = new RatedRequest("berlin_query", createRelevant("index", "berlin"), testQuery); List specifications = new ArrayList<>(); specifications.add(amsterdamRequest); specifications.add(berlinRequest); @@ -95,7 +90,7 @@ public void testRankEvalRequest() throws IOException { assertEquals(7, hitsAndRatings.size()); for (RatedSearchHit hit : hitsAndRatings) { String id = hit.getSearchHit().getId(); - if (id.equals("1") || id.equals("6")) { + if (id.equals("berlin") || id.equals("amsterdam5")) { assertFalse(hit.getRating().isPresent()); } else { assertEquals(1, hit.getRating().get().intValue()); @@ -107,7 +102,7 @@ public void testRankEvalRequest() throws IOException { assertEquals(7, hitsAndRatings.size()); for (RatedSearchHit hit : hitsAndRatings) { String id = hit.getSearchHit().getId(); - if (id.equals("1")) { + if (id.equals("berlin")) { assertEquals(1, hit.getRating().get().intValue()); } else { assertFalse(hit.getRating().isPresent()); @@ -115,7 +110,7 @@ public void testRankEvalRequest() throws IOException { } // now try this when test2 is closed - client().performRequest("POST", "index2/_close", Collections.emptyMap()); + client().performRequest(new Request("POST", "index2/_close")); rankEvalRequest.indicesOptions(IndicesOptions.fromParameters(null, "true", null, SearchRequest.DEFAULT_INDICES_OPTIONS)); response = execute(rankEvalRequest, highLevelClient()::rankEval, highLevelClient()::rankEvalAsync, highLevelClient()::rankEval, highLevelClient()::rankEvalAsync); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java index 469b55c785bed..c436e7c727ea5 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java @@ -19,12 +19,8 @@ package org.elasticsearch.client; -import org.apache.http.HttpEntity; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPut; -import org.apache.http.entity.ContentType; -import org.apache.http.entity.StringEntity; -import org.apache.http.nio.entity.NStringEntity; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.explain.ExplainRequest; @@ -101,85 +97,106 @@ public class SearchIT extends ESRestHighLevelClientTestCase { @Before public void indexDocuments() throws IOException { - StringEntity doc1 = new StringEntity("{\"type\":\"type1\", \"num\":10, \"num2\":50}", ContentType.APPLICATION_JSON); - client().performRequest(HttpPut.METHOD_NAME, "/index/type/1", Collections.emptyMap(), doc1); - StringEntity doc2 = new StringEntity("{\"type\":\"type1\", \"num\":20, \"num2\":40}", ContentType.APPLICATION_JSON); - client().performRequest(HttpPut.METHOD_NAME, "/index/type/2", Collections.emptyMap(), doc2); - StringEntity doc3 = new StringEntity("{\"type\":\"type1\", \"num\":50, \"num2\":35}", ContentType.APPLICATION_JSON); - client().performRequest(HttpPut.METHOD_NAME, "/index/type/3", Collections.emptyMap(), doc3); - StringEntity doc4 = new StringEntity("{\"type\":\"type2\", \"num\":100, \"num2\":10}", ContentType.APPLICATION_JSON); - client().performRequest(HttpPut.METHOD_NAME, "/index/type/4", Collections.emptyMap(), doc4); - StringEntity doc5 = new StringEntity("{\"type\":\"type2\", \"num\":100, \"num2\":10}", ContentType.APPLICATION_JSON); - client().performRequest(HttpPut.METHOD_NAME, "/index/type/5", Collections.emptyMap(), doc5); - client().performRequest(HttpPost.METHOD_NAME, "/index/_refresh"); - - - StringEntity doc = new StringEntity("{\"field\":\"value1\", \"rating\": 7}", ContentType.APPLICATION_JSON); - client().performRequest(HttpPut.METHOD_NAME, "/index1/doc/1", Collections.emptyMap(), doc); - doc = new StringEntity("{\"field\":\"value2\"}", ContentType.APPLICATION_JSON); - client().performRequest(HttpPut.METHOD_NAME, "/index1/doc/2", Collections.emptyMap(), doc); - - StringEntity mappings = new StringEntity( - "{" + - " \"mappings\": {" + - " \"doc\": {" + - " \"properties\": {" + - " \"rating\": {" + - " \"type\": \"keyword\"" + - " }" + - " }" + - " }" + - " }" + - "}}", - ContentType.APPLICATION_JSON); - client().performRequest("PUT", "/index2", Collections.emptyMap(), mappings); - doc = new StringEntity("{\"field\":\"value1\", \"rating\": \"good\"}", ContentType.APPLICATION_JSON); - client().performRequest(HttpPut.METHOD_NAME, "/index2/doc/3", Collections.emptyMap(), doc); - doc = new StringEntity("{\"field\":\"value2\"}", ContentType.APPLICATION_JSON); - client().performRequest(HttpPut.METHOD_NAME, "/index2/doc/4", Collections.emptyMap(), doc); - - doc = new StringEntity("{\"field\":\"value1\"}", ContentType.APPLICATION_JSON); - client().performRequest(HttpPut.METHOD_NAME, "/index3/doc/5", Collections.emptyMap(), doc); - doc = new StringEntity("{\"field\":\"value2\"}", ContentType.APPLICATION_JSON); - client().performRequest(HttpPut.METHOD_NAME, "/index3/doc/6", Collections.emptyMap(), doc); - - mappings = new StringEntity( - "{" + + { + Request doc1 = new Request(HttpPut.METHOD_NAME, "/index/type/1"); + doc1.setJsonEntity("{\"type\":\"type1\", \"num\":10, \"num2\":50}"); + client().performRequest(doc1); + Request doc2 = new Request(HttpPut.METHOD_NAME, "/index/type/2"); + doc2.setJsonEntity("{\"type\":\"type1\", \"num\":20, \"num2\":40}"); + client().performRequest(doc2); + Request doc3 = new Request(HttpPut.METHOD_NAME, "/index/type/3"); + doc3.setJsonEntity("{\"type\":\"type1\", \"num\":50, \"num2\":35}"); + client().performRequest(doc3); + Request doc4 = new Request(HttpPut.METHOD_NAME, "/index/type/4"); + doc4.setJsonEntity("{\"type\":\"type2\", \"num\":100, \"num2\":10}"); + client().performRequest(doc4); + Request doc5 = new Request(HttpPut.METHOD_NAME, "/index/type/5"); + doc5.setJsonEntity("{\"type\":\"type2\", \"num\":100, \"num2\":10}"); + client().performRequest(doc5); + } + + { + Request doc1 = new Request(HttpPut.METHOD_NAME, "/index1/doc/1"); + doc1.setJsonEntity("{\"field\":\"value1\", \"rating\": 7}"); + client().performRequest(doc1); + Request doc2 = new Request(HttpPut.METHOD_NAME, "/index1/doc/2"); + doc2.setJsonEntity("{\"field\":\"value2\"}"); + client().performRequest(doc2); + } + + { + Request create = new Request("PUT", "/index2"); + create.setJsonEntity( + "{" + " \"mappings\": {" + " \"doc\": {" + " \"properties\": {" + - " \"field1\": {" + - " \"type\": \"keyword\"," + - " \"store\": true" + - " }," + - " \"field2\": {" + - " \"type\": \"keyword\"," + - " \"store\": true" + + " \"rating\": {" + + " \"type\": \"keyword\"" + " }" + " }" + " }" + " }" + - "}}", - ContentType.APPLICATION_JSON); - client().performRequest(HttpPut.METHOD_NAME, "/index4", Collections.emptyMap(), mappings); - doc = new StringEntity("{\"field1\":\"value1\", \"field2\":\"value2\"}", ContentType.APPLICATION_JSON); - client().performRequest(HttpPut.METHOD_NAME, "/index4/doc/1", Collections.emptyMap(), doc); - StringEntity aliasFilter = new StringEntity( - "{" + - " \"actions\" : [" + - " {" + - " \"add\" : {" + - " \"index\" : \"index4\"," + - " \"alias\" : \"alias4\"," + - " \"filter\" : { \"term\" : { \"field2\" : \"value1\" } }" + - " }" + - " }" + - " ]" + - "}", - ContentType.APPLICATION_JSON); - client().performRequest(HttpPost.METHOD_NAME, "/_aliases", Collections.emptyMap(), aliasFilter); + "}"); + client().performRequest(create); + Request doc3 = new Request(HttpPut.METHOD_NAME, "/index2/doc/3"); + doc3.setJsonEntity("{\"field\":\"value1\", \"rating\": \"good\"}"); + client().performRequest(doc3); + Request doc4 = new Request(HttpPut.METHOD_NAME, "/index2/doc/4"); + doc4.setJsonEntity("{\"field\":\"value2\"}"); + client().performRequest(doc4); + } + + { + Request doc5 = new Request(HttpPut.METHOD_NAME, "/index3/doc/5"); + doc5.setJsonEntity("{\"field\":\"value1\"}"); + client().performRequest(doc5); + Request doc6 = new Request(HttpPut.METHOD_NAME, "/index3/doc/6"); + doc6.setJsonEntity("{\"field\":\"value2\"}"); + client().performRequest(doc6); + } + + { + Request create = new Request(HttpPut.METHOD_NAME, "/index4"); + create.setJsonEntity( + "{" + + " \"mappings\": {" + + " \"doc\": {" + + " \"properties\": {" + + " \"field1\": {" + + " \"type\": \"keyword\"," + + " \"store\": true" + + " }," + + " \"field2\": {" + + " \"type\": \"keyword\"," + + " \"store\": true" + + " }" + + " }" + + " }" + + " }" + + "}"); + client().performRequest(create); + Request doc1 = new Request(HttpPut.METHOD_NAME, "/index4/doc/1"); + doc1.setJsonEntity("{\"field1\":\"value1\", \"field2\":\"value2\"}"); + client().performRequest(doc1); + + Request createFilteredAlias = new Request(HttpPost.METHOD_NAME, "/_aliases"); + createFilteredAlias.setJsonEntity( + "{" + + " \"actions\" : [" + + " {" + + " \"add\" : {" + + " \"index\" : \"index4\"," + + " \"alias\" : \"alias4\"," + + " \"filter\" : { \"term\" : { \"field2\" : \"value1\" } }" + + " }" + + " }" + + " ]" + + "}"); + client().performRequest(createFilteredAlias); + } - client().performRequest(HttpPost.METHOD_NAME, "/index1,index2,index3,index4/_refresh"); + client().performRequest(new Request(HttpPost.METHOD_NAME, "/_refresh")); } public void testSearchNoQuery() throws IOException { @@ -383,7 +400,9 @@ public void testSearchWithMatrixStats() throws IOException { public void testSearchWithParentJoin() throws IOException { final String indexName = "child_example"; - StringEntity parentMapping = new StringEntity("{\n" + + Request createIndex = new Request(HttpPut.METHOD_NAME, "/" + indexName); + createIndex.setJsonEntity( + "{\n" + " \"mappings\": {\n" + " \"qa\" : {\n" + " \"properties\" : {\n" + @@ -394,9 +413,11 @@ public void testSearchWithParentJoin() throws IOException { " }\n" + " }\n" + " }" + - "}", ContentType.APPLICATION_JSON); - client().performRequest(HttpPut.METHOD_NAME, "/" + indexName, Collections.emptyMap(), parentMapping); - StringEntity questionDoc = new StringEntity("{\n" + + "}"); + client().performRequest(createIndex); + Request questionDoc = new Request(HttpPut.METHOD_NAME, "/" + indexName + "/qa/1"); + questionDoc.setJsonEntity( + "{\n" + " \"body\": \"

    I have Windows 2003 server and i bought a new Windows 2008 server...\",\n" + " \"title\": \"Whats the best way to file transfer my site from server to a newer one?\",\n" + " \"tags\": [\n" + @@ -405,9 +426,12 @@ public void testSearchWithParentJoin() throws IOException { " \"file-transfer\"\n" + " ],\n" + " \"qa_join_field\" : \"question\"\n" + - "}", ContentType.APPLICATION_JSON); - client().performRequest(HttpPut.METHOD_NAME, "/" + indexName + "/qa/1", Collections.emptyMap(), questionDoc); - StringEntity answerDoc1 = new StringEntity("{\n" + + "}"); + client().performRequest(questionDoc); + Request answerDoc1 = new Request(HttpPut.METHOD_NAME, "/" + indexName + "/qa/2"); + answerDoc1.addParameter("routing", "1"); + answerDoc1.setJsonEntity( + "{\n" + " \"owner\": {\n" + " \"location\": \"Norfolk, United Kingdom\",\n" + " \"display_name\": \"Sam\",\n" + @@ -419,9 +443,12 @@ public void testSearchWithParentJoin() throws IOException { " \"parent\" : \"1\"\n" + " },\n" + " \"creation_date\": \"2009-05-04T13:45:37.030\"\n" + - "}", ContentType.APPLICATION_JSON); - client().performRequest(HttpPut.METHOD_NAME, "/" + indexName + "/qa/2", Collections.singletonMap("routing", "1"), answerDoc1); - StringEntity answerDoc2 = new StringEntity("{\n" + + "}"); + client().performRequest(answerDoc1); + Request answerDoc2 = new Request(HttpPut.METHOD_NAME, "/" + indexName + "/qa/3"); + answerDoc2.addParameter("routing", "1"); + answerDoc2.setJsonEntity( + "{\n" + " \"owner\": {\n" + " \"location\": \"Norfolk, United Kingdom\",\n" + " \"display_name\": \"Troll\",\n" + @@ -433,9 +460,9 @@ public void testSearchWithParentJoin() throws IOException { " \"parent\" : \"1\"\n" + " },\n" + " \"creation_date\": \"2009-05-05T13:45:37.030\"\n" + - "}", ContentType.APPLICATION_JSON); - client().performRequest(HttpPut.METHOD_NAME, "/" + indexName + "/qa/3", Collections.singletonMap("routing", "1"), answerDoc2); - client().performRequest(HttpPost.METHOD_NAME, "/_refresh"); + "}"); + client().performRequest(answerDoc2); + client().performRequest(new Request(HttpPost.METHOD_NAME, "/_refresh")); TermsAggregationBuilder leafTermAgg = new TermsAggregationBuilder("top-names", ValueType.STRING) .field("owner.display_name.keyword").size(10); @@ -514,9 +541,10 @@ public void testSearchWithSuggest() throws IOException { } public void testSearchWithWeirdScriptFields() throws Exception { - HttpEntity entity = new NStringEntity("{ \"field\":\"value\"}", ContentType.APPLICATION_JSON); - client().performRequest("PUT", "test/type/1", Collections.emptyMap(), entity); - client().performRequest("POST", "/test/_refresh"); + Request doc = new Request("PUT", "test/type/1"); + doc.setJsonEntity("{\"field\":\"value\"}"); + client().performRequest(doc); + client().performRequest(new Request("POST", "/test/_refresh")); { SearchRequest searchRequest = new SearchRequest("test").source(SearchSourceBuilder.searchSource() @@ -558,13 +586,13 @@ public void testSearchWithWeirdScriptFields() throws Exception { } public void testSearchScroll() throws Exception { - for (int i = 0; i < 100; i++) { XContentBuilder builder = jsonBuilder().startObject().field("field", i).endObject(); - HttpEntity entity = new NStringEntity(Strings.toString(builder), ContentType.APPLICATION_JSON); - client().performRequest(HttpPut.METHOD_NAME, "test/type1/" + Integer.toString(i), Collections.emptyMap(), entity); + Request doc = new Request(HttpPut.METHOD_NAME, "/test/type1/" + Integer.toString(i)); + doc.setJsonEntity(Strings.toString(builder)); + client().performRequest(doc); } - client().performRequest(HttpPost.METHOD_NAME, "/test/_refresh"); + client().performRequest(new Request(HttpPost.METHOD_NAME, "/test/_refresh")); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().size(35).sort("field", SortOrder.ASC); SearchRequest searchRequest = new SearchRequest("test").scroll(TimeValue.timeValueMinutes(2)).source(searchSourceBuilder); @@ -893,11 +921,11 @@ public void testRenderSearchTemplate() throws IOException { assertToXContentEquivalent(expectedSource, actualSource, XContentType.JSON); } - - + + public void testMultiSearchTemplate() throws Exception { MultiSearchTemplateRequest multiSearchTemplateRequest = new MultiSearchTemplateRequest(); - + SearchTemplateRequest goodRequest = new SearchTemplateRequest(); goodRequest.setRequest(new SearchRequest("index")); goodRequest.setScriptType(ScriptType.INLINE); @@ -915,8 +943,8 @@ public void testMultiSearchTemplate() throws Exception { goodRequest.setExplain(true); goodRequest.setProfile(true); multiSearchTemplateRequest.add(goodRequest); - - + + SearchTemplateRequest badRequest = new SearchTemplateRequest(); badRequest.setRequest(new SearchRequest("index")); badRequest.setScriptType(ScriptType.INLINE); @@ -925,17 +953,17 @@ public void testMultiSearchTemplate() throws Exception { scriptParams.put("number", 10); badRequest.setScriptParams(scriptParams); - multiSearchTemplateRequest.add(badRequest); - + multiSearchTemplateRequest.add(badRequest); + MultiSearchTemplateResponse multiSearchTemplateResponse = - execute(multiSearchTemplateRequest, highLevelClient()::multiSearchTemplate, + execute(multiSearchTemplateRequest, highLevelClient()::multiSearchTemplate, highLevelClient()::multiSearchTemplateAsync); - + Item[] responses = multiSearchTemplateResponse.getResponses(); - + assertEquals(2, responses.length); - - + + assertNull(responses[0].getResponse().getSource()); SearchResponse goodResponse =responses[0].getResponse().getResponse(); assertNotNull(goodResponse); @@ -945,18 +973,18 @@ public void testMultiSearchTemplate() throws Exception { assertThat(goodResponse.getHits().getMaxScore(), greaterThan(0f)); SearchHit hit = goodResponse.getHits().getHits()[0]; assertNotNull(hit.getExplanation()); - assertFalse(goodResponse.getProfileResults().isEmpty()); - - + assertFalse(goodResponse.getProfileResults().isEmpty()); + + assertNull(responses[0].getResponse().getSource()); assertThat(responses[1].isFailure(), Matchers.is(true)); - assertNotNull(responses[1].getFailureMessage()); + assertNotNull(responses[1].getFailureMessage()); assertThat(responses[1].getFailureMessage(), containsString("json_parse_exception")); } - + public void testMultiSearchTemplateAllBad() throws Exception { MultiSearchTemplateRequest multiSearchTemplateRequest = new MultiSearchTemplateRequest(); - + SearchTemplateRequest badRequest1 = new SearchTemplateRequest(); badRequest1.setRequest(new SearchRequest("index")); badRequest1.setScriptType(ScriptType.INLINE); @@ -972,8 +1000,8 @@ public void testMultiSearchTemplateAllBad() throws Exception { scriptParams.put("number", "BAD NUMBER"); badRequest1.setScriptParams(scriptParams); multiSearchTemplateRequest.add(badRequest1); - - + + SearchTemplateRequest badRequest2 = new SearchTemplateRequest(); badRequest2.setRequest(new SearchRequest("index")); badRequest2.setScriptType(ScriptType.INLINE); @@ -982,13 +1010,13 @@ public void testMultiSearchTemplateAllBad() throws Exception { scriptParams.put("number", "BAD NUMBER"); badRequest2.setScriptParams(scriptParams); - multiSearchTemplateRequest.add(badRequest2); - - // The whole HTTP request should fail if no nested search requests are valid + multiSearchTemplateRequest.add(badRequest2); + + // The whole HTTP request should fail if no nested search requests are valid ElasticsearchStatusException exception = expectThrows(ElasticsearchStatusException.class, - () -> execute(multiSearchTemplateRequest, highLevelClient()::multiSearchTemplate, + () -> execute(multiSearchTemplateRequest, highLevelClient()::multiSearchTemplate, highLevelClient()::multiSearchTemplateAsync)); - + assertEquals(RestStatus.BAD_REQUEST, exception.status()); assertThat(exception.getMessage(), containsString("no requests added")); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java index c706a3a03f203..044d8fdd04997 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/CRUDDocumentationIT.java @@ -19,8 +19,6 @@ package org.elasticsearch.client.documentation; -import org.apache.http.entity.ContentType; -import org.apache.http.nio.entity.NStringEntity; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.DocWriteRequest; @@ -66,7 +64,6 @@ import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; -import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.Map; @@ -766,7 +763,9 @@ public void onFailure(Exception e) { public void testGet() throws Exception { RestHighLevelClient client = highLevelClient(); { - String mappings = "{\n" + + Request createIndex = new Request("PUT", "/posts"); + createIndex.setJsonEntity( + "{\n" + " \"mappings\" : {\n" + " \"doc\" : {\n" + " \"properties\" : {\n" + @@ -777,10 +776,8 @@ public void testGet() throws Exception { " }\n" + " }\n" + " }\n" + - "}"; - - NStringEntity entity = new NStringEntity(mappings, ContentType.APPLICATION_JSON); - Response response = client().performRequest("PUT", "/posts", Collections.emptyMap(), entity); + "}"); + Response response = client().performRequest(createIndex); assertEquals(200, response.getStatusLine().getStatusCode()); IndexRequest indexRequest = new IndexRequest("posts", "doc", "1") @@ -1085,21 +1082,21 @@ public void testMultiGet() throws Exception { RestHighLevelClient client = highLevelClient(); { - String mappings = "{\n" + - " \"mappings\" : {\n" + - " \"type\" : {\n" + - " \"properties\" : {\n" + - " \"foo\" : {\n" + - " \"type\": \"text\",\n" + - " \"store\": true\n" + - " }\n" + - " }\n" + - " }\n" + - " }\n" + - "}"; - - NStringEntity entity = new NStringEntity(mappings, ContentType.APPLICATION_JSON); - Response response = client().performRequest("PUT", "/index", Collections.emptyMap(), entity); + Request createIndex = new Request("PUT", "/index"); + createIndex.setJsonEntity( + "{\n" + + " \"mappings\" : {\n" + + " \"type\" : {\n" + + " \"properties\" : {\n" + + " \"foo\" : {\n" + + " \"type\": \"text\",\n" + + " \"store\": true\n" + + " }\n" + + " }\n" + + " }\n" + + " }\n" + + "}"); + Response response = client().performRequest(createIndex); assertEquals(200, response.getStatusLine().getStatusCode()); } From b73e9f191d5752d674cec002b4c730d39bd7928a Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 11 Jul 2018 09:48:47 -0400 Subject: [PATCH 24/50] Switch low level rest tests to new style Requests (#31938) In #29623 we added `Request` object flavored requests to the low level REST client and in #30315 we deprecated the old `performRequest`s. This changes all calls in the `client/rest` project to use the new versions. --- .../client/RestClientBuilderIntegTests.java | 4 +- .../RestClientSingleHostIntegTests.java | 38 +++++++++++++------ .../RestClientDocumentation.java | 2 +- 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderIntegTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderIntegTests.java index 199b7542e62a2..93f8481bea662 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderIntegTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderIntegTests.java @@ -76,7 +76,7 @@ public void testBuilderUsesDefaultSSLContext() throws Exception { try { try (RestClient client = buildRestClient()) { try { - client.performRequest("GET", "/"); + client.performRequest(new Request("GET", "/")); fail("connection should have been rejected due to SSL handshake"); } catch (Exception e) { assertThat(e.getMessage(), containsString("General SSLEngine problem")); @@ -85,7 +85,7 @@ public void testBuilderUsesDefaultSSLContext() throws Exception { SSLContext.setDefault(getSslContext()); try (RestClient client = buildRestClient()) { - Response response = client.performRequest("GET", "/"); + Response response = client.performRequest(new Request("GET", "/")); assertEquals(200, response.getStatusLine().getStatusCode()); } } finally { diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostIntegTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostIntegTests.java index 114d34c73da89..6b5bb3c98ee5b 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostIntegTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientSingleHostIntegTests.java @@ -256,35 +256,51 @@ public void testGetWithBody() throws IOException { public void testEncodeParams() throws IOException { { - Response response = restClient.performRequest("PUT", "/200", Collections.singletonMap("routing", "this/is/the/routing")); + Request request = new Request("PUT", "/200"); + request.addParameter("routing", "this/is/the/routing"); + Response response = restClient.performRequest(request); assertEquals(pathPrefix + "/200?routing=this%2Fis%2Fthe%2Frouting", response.getRequestLine().getUri()); } { - Response response = restClient.performRequest("PUT", "/200", Collections.singletonMap("routing", "this|is|the|routing")); + Request request = new Request("PUT", "/200"); + request.addParameter("routing", "this|is|the|routing"); + Response response = restClient.performRequest(request); assertEquals(pathPrefix + "/200?routing=this%7Cis%7Cthe%7Crouting", response.getRequestLine().getUri()); } { - Response response = restClient.performRequest("PUT", "/200", Collections.singletonMap("routing", "routing#1")); + Request request = new Request("PUT", "/200"); + request.addParameter("routing", "routing#1"); + Response response = restClient.performRequest(request); assertEquals(pathPrefix + "/200?routing=routing%231", response.getRequestLine().getUri()); } { - Response response = restClient.performRequest("PUT", "/200", Collections.singletonMap("routing", "中文")); + Request request = new Request("PUT", "/200"); + request.addParameter("routing", "中文"); + Response response = restClient.performRequest(request); assertEquals(pathPrefix + "/200?routing=%E4%B8%AD%E6%96%87", response.getRequestLine().getUri()); } { - Response response = restClient.performRequest("PUT", "/200", Collections.singletonMap("routing", "foo bar")); + Request request = new Request("PUT", "/200"); + request.addParameter("routing", "foo bar"); + Response response = restClient.performRequest(request); assertEquals(pathPrefix + "/200?routing=foo+bar", response.getRequestLine().getUri()); } { - Response response = restClient.performRequest("PUT", "/200", Collections.singletonMap("routing", "foo+bar")); + Request request = new Request("PUT", "/200"); + request.addParameter("routing", "foo+bar"); + Response response = restClient.performRequest(request); assertEquals(pathPrefix + "/200?routing=foo%2Bbar", response.getRequestLine().getUri()); } { - Response response = restClient.performRequest("PUT", "/200", Collections.singletonMap("routing", "foo/bar")); + Request request = new Request("PUT", "/200"); + request.addParameter("routing", "foo/bar"); + Response response = restClient.performRequest(request); assertEquals(pathPrefix + "/200?routing=foo%2Fbar", response.getRequestLine().getUri()); } { - Response response = restClient.performRequest("PUT", "/200", Collections.singletonMap("routing", "foo^bar")); + Request request = new Request("PUT", "/200"); + request.addParameter("routing", "foo^bar"); + Response response = restClient.performRequest(request); assertEquals(pathPrefix + "/200?routing=foo%5Ebar", response.getRequestLine().getUri()); } } @@ -341,14 +357,14 @@ public void testAuthCredentialsAreNotClearedOnAuthChallenge() throws IOException public void testUrlWithoutLeadingSlash() throws Exception { if (pathPrefix.length() == 0) { try { - restClient.performRequest("GET", "200"); + restClient.performRequest(new Request("GET", "200")); fail("request should have failed"); } catch (ResponseException e) { assertEquals(404, e.getResponse().getStatusLine().getStatusCode()); } } else { { - Response response = restClient.performRequest("GET", "200"); + Response response = restClient.performRequest(new Request("GET", "200")); //a trailing slash gets automatically added if a pathPrefix is configured assertEquals(200, response.getStatusLine().getStatusCode()); } @@ -357,7 +373,7 @@ public void testUrlWithoutLeadingSlash() throws Exception { try (RestClient restClient = RestClient.builder( new HttpHost(httpServer.getAddress().getHostString(), httpServer.getAddress().getPort())) .setPathPrefix(pathPrefix.substring(1)).build()) { - Response response = restClient.performRequest("GET", "200"); + Response response = restClient.performRequest(new Request("GET", "200")); //a trailing slash gets automatically added if a pathPrefix is configured assertEquals(200, response.getStatusLine().getStatusCode()); } diff --git a/client/rest/src/test/java/org/elasticsearch/client/documentation/RestClientDocumentation.java b/client/rest/src/test/java/org/elasticsearch/client/documentation/RestClientDocumentation.java index d347353a1fb55..ce2e0907560cd 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/documentation/RestClientDocumentation.java +++ b/client/rest/src/test/java/org/elasticsearch/client/documentation/RestClientDocumentation.java @@ -267,7 +267,7 @@ public void onFailure(Exception exception) { } { //tag::rest-client-response2 - Response response = restClient.performRequest("GET", "/"); + Response response = restClient.performRequest(new Request("GET", "/")); RequestLine requestLine = response.getRequestLine(); // <1> HttpHost host = response.getHost(); // <2> int statusCode = response.getStatusLine().getStatusCode(); // <3> From 60c58701575b2fbcac74198968a9166a52f8cc22 Mon Sep 17 00:00:00 2001 From: albendz <12453997+albendz@users.noreply.github.com> Date: Tue, 10 Jul 2018 07:47:00 -0700 Subject: [PATCH 25/50] Watcher: Slack message empty text (#31596) Slack accepts an empty text or attachments, but not both. This commit ensures that both are not empty when creating a watch. Closes #30071 Replacing old pull request: #31288 --- .../slack/message/SlackMessage.java | 7 ++++++- .../slack/message/SlackMessageTests.java | 18 +++++++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/slack/message/SlackMessage.java b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/slack/message/SlackMessage.java index 9f7cd36c9106a..ffff28ce862bd 100644 --- a/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/slack/message/SlackMessage.java +++ b/x-pack/plugin/watcher/src/main/java/org/elasticsearch/xpack/watcher/notification/slack/message/SlackMessage.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.xpack.watcher.common.text.TextTemplate; import org.elasticsearch.xpack.watcher.common.text.TextTemplateEngine; +import org.elasticsearch.common.Nullable; import java.io.IOException; import java.util.ArrayList; @@ -29,7 +30,11 @@ public class SlackMessage implements MessageElement { final String text; final Attachment[] attachments; - public SlackMessage(String from, String[] to, String icon, String text, Attachment[] attachments) { + public SlackMessage(String from, String[] to, String icon, @Nullable String text, @Nullable Attachment[] attachments) { + if(text == null && attachments == null) { + throw new IllegalArgumentException("Both text and attachments cannot be null."); + } + this.from = from; this.to = to; this.icon = icon; diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/slack/message/SlackMessageTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/slack/message/SlackMessageTests.java index 0432fa41d864c..b41e58b06126c 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/slack/message/SlackMessageTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/slack/message/SlackMessageTests.java @@ -49,7 +49,7 @@ public void testToXContent() throws Exception { } String icon = randomBoolean() ? null : randomAlphaOfLength(10); String text = randomBoolean() ? null : randomAlphaOfLength(50); - Attachment[] attachments = randomBoolean() ? null : new Attachment[randomIntBetween(0, 2)]; + Attachment[] attachments = (text != null && randomBoolean()) ? null : new Attachment[randomIntBetween(0, 2)]; if (attachments != null) { for (int i = 0; i < attachments.length; i++) { String fallback = randomBoolean() ? null : randomAlphaOfLength(10); @@ -600,6 +600,22 @@ public void testUrlPathIsFiltered() throws Exception { } } + public void testCanHaveNullText() throws Exception { + SlackMessage slackMessage = new SlackMessage("from", new String[] {"to"}, "icon", null, new Attachment[1]); + assertNull(slackMessage.getText()); + assertNotNull(slackMessage.getAttachments()); + } + + public void testCanHaveNullAttachments() throws Exception { + SlackMessage slackMessage = new SlackMessage("from", new String[] {"to"}, "icon", "text", null); + assertNotNull(slackMessage.getText()); + assertNull(slackMessage.getAttachments()); + } + + public void testCannotHaveNullAttachmentsAndNullText() throws Exception { + expectThrows(IllegalArgumentException.class, () -> new SlackMessage("from", new String[]{"to"}, "icon", null, null)); + } + private static void writeFieldIfNotNull(XContentBuilder builder, String field, Object value) throws IOException { if (value != null) { builder.field(field, value); From 9b7439bfc3697d11c9d7b5055833852e6e912761 Mon Sep 17 00:00:00 2001 From: Michael Basnight Date: Wed, 11 Jul 2018 09:37:48 -0500 Subject: [PATCH 26/50] Test: fix null failure in watcher test (#31968) A new commit was merged that does not allow a null attachement && text. This is valid for the slack API, as it does not allow this, but our unit tests did. This commit fixes the broken unit test. Closes #31948 --- .../watcher/notification/slack/message/SlackMessageTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/slack/message/SlackMessageTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/slack/message/SlackMessageTests.java index b41e58b06126c..83e2e9978391a 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/slack/message/SlackMessageTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/slack/message/SlackMessageTests.java @@ -481,7 +481,7 @@ public void testTemplateRender() throws Exception { if (randomBoolean()) { templateBuilder.setText(randomAlphaOfLength(10)); } - if (randomBoolean()) { + if (templateBuilder.text == null || randomBoolean()) { int count = randomIntBetween(0, 3); for (int i = 0; i < count; i++) { Attachment.Template.Builder attachmentBuilder = createRandomAttachmentTemplateBuilder(); From f19d83fc291b885b2e1ef9933d15e1a3120501cd Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Wed, 11 Jul 2018 10:13:41 -0500 Subject: [PATCH 27/50] ingest: date_index_name processor template resolution (#31841) This change adds support for template snippet (e.g. {{foo}}) resolution in the date_index_name processor. The following configuration options will now resolve a templated value if so configured: * index_name_prefix (e.g "index_name_prefix": "myindex-{{foo}}-") * date_rounding (e.g. "date_rounding" : "{{bar}}") * index_name_format (e.g."index_name_format": "{{baz}}") --- .../ingest/common/DateIndexNameProcessor.java | 47 +++++++++++++------ .../ingest/common/IngestCommonPlugin.java | 6 +-- .../common/DateIndexNameFactoryTests.java | 19 ++++---- .../common/DateIndexNameProcessorTests.java | 45 ++++++++++++++---- 4 files changed, 83 insertions(+), 34 deletions(-) diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java index b44eaa3bfa3ca..0d6253c88f9fa 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/DateIndexNameProcessor.java @@ -32,6 +32,8 @@ import org.elasticsearch.ingest.ConfigurationUtils; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.Processor; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.TemplateScript; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; import org.joda.time.format.DateTimeFormat; @@ -42,21 +44,22 @@ public final class DateIndexNameProcessor extends AbstractProcessor { public static final String TYPE = "date_index_name"; private final String field; - private final String indexNamePrefix; - private final String dateRounding; - private final String indexNameFormat; + private final TemplateScript.Factory indexNamePrefixTemplate; + private final TemplateScript.Factory dateRoundingTemplate; + private final TemplateScript.Factory indexNameFormatTemplate; private final DateTimeZone timezone; private final List> dateFormats; DateIndexNameProcessor(String tag, String field, List> dateFormats, DateTimeZone timezone, - String indexNamePrefix, String dateRounding, String indexNameFormat) { + TemplateScript.Factory indexNamePrefixTemplate, TemplateScript.Factory dateRoundingTemplate, + TemplateScript.Factory indexNameFormatTemplate) { super(tag); this.field = field; this.timezone = timezone; this.dateFormats = dateFormats; - this.indexNamePrefix = indexNamePrefix; - this.dateRounding = dateRounding; - this.indexNameFormat = indexNameFormat; + this.indexNamePrefixTemplate = indexNamePrefixTemplate; + this.dateRoundingTemplate = dateRoundingTemplate; + this.indexNameFormatTemplate = indexNameFormatTemplate; } @Override @@ -83,6 +86,9 @@ public void execute(IngestDocument ingestDocument) throws Exception { if (dateTime == null) { throw new IllegalArgumentException("unable to parse date [" + date + "]", lastException); } + String indexNamePrefix = ingestDocument.renderTemplate(indexNamePrefixTemplate); + String indexNameFormat = ingestDocument.renderTemplate(indexNameFormatTemplate); + String dateRounding = ingestDocument.renderTemplate(dateRoundingTemplate); DateTimeFormatter formatter = DateTimeFormat.forPattern(indexNameFormat); StringBuilder builder = new StringBuilder() @@ -106,16 +112,16 @@ String getField() { return field; } - String getIndexNamePrefix() { - return indexNamePrefix; + TemplateScript.Factory getIndexNamePrefixTemplate() { + return indexNamePrefixTemplate; } - String getDateRounding() { - return dateRounding; + TemplateScript.Factory getDateRoundingTemplate() { + return dateRoundingTemplate; } - String getIndexNameFormat() { - return indexNameFormat; + TemplateScript.Factory getIndexNameFormatTemplate() { + return indexNameFormatTemplate; } DateTimeZone getTimezone() { @@ -128,6 +134,12 @@ List> getDateFormats() { public static final class Factory implements Processor.Factory { + private final ScriptService scriptService; + + public Factory(ScriptService scriptService) { + this.scriptService = scriptService; + } + @Override public DateIndexNameProcessor create(Map registry, String tag, Map config) throws Exception { @@ -154,9 +166,16 @@ public DateIndexNameProcessor create(Map registry, St String field = ConfigurationUtils.readStringProperty(TYPE, tag, config, "field"); String indexNamePrefix = ConfigurationUtils.readStringProperty(TYPE, tag, config, "index_name_prefix", ""); + TemplateScript.Factory indexNamePrefixTemplate = + ConfigurationUtils.compileTemplate(TYPE, tag, "index_name_prefix", indexNamePrefix, scriptService); String dateRounding = ConfigurationUtils.readStringProperty(TYPE, tag, config, "date_rounding"); + TemplateScript.Factory dateRoundingTemplate = + ConfigurationUtils.compileTemplate(TYPE, tag, "date_rounding", dateRounding, scriptService); String indexNameFormat = ConfigurationUtils.readStringProperty(TYPE, tag, config, "index_name_format", "yyyy-MM-dd"); - return new DateIndexNameProcessor(tag, field, dateFormats, timezone, indexNamePrefix, dateRounding, indexNameFormat); + TemplateScript.Factory indexNameFormatTemplate = + ConfigurationUtils.compileTemplate(TYPE, tag, "index_name_format", indexNameFormat, scriptService); + return new DateIndexNameProcessor(tag, field, dateFormats, timezone, indexNamePrefixTemplate, + dateRoundingTemplate, indexNameFormatTemplate); } } diff --git a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java index d9878cae9e2ab..bc475a2a00539 100644 --- a/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java +++ b/modules/ingest-common/src/main/java/org/elasticsearch/ingest/common/IngestCommonPlugin.java @@ -73,7 +73,7 @@ public Map getProcessors(Processor.Parameters paramet processors.put(GsubProcessor.TYPE, new GsubProcessor.Factory()); processors.put(FailProcessor.TYPE, new FailProcessor.Factory(parameters.scriptService)); processors.put(ForEachProcessor.TYPE, new ForEachProcessor.Factory()); - processors.put(DateIndexNameProcessor.TYPE, new DateIndexNameProcessor.Factory()); + processors.put(DateIndexNameProcessor.TYPE, new DateIndexNameProcessor.Factory(parameters.scriptService)); processors.put(SortProcessor.TYPE, new SortProcessor.Factory()); processors.put(GrokProcessor.TYPE, new GrokProcessor.Factory(GROK_PATTERNS, createGrokThreadWatchdog(parameters))); processors.put(ScriptProcessor.TYPE, new ScriptProcessor.Factory(parameters.scriptService)); @@ -97,12 +97,12 @@ public List getRestHandlers(Settings settings, RestController restC Supplier nodesInCluster) { return Arrays.asList(new GrokProcessorGetAction.RestAction(settings, restController)); } - + @Override public List> getSettings() { return Arrays.asList(WATCHDOG_INTERVAL, WATCHDOG_MAX_EXECUTION_TIME); } - + private static ThreadWatchdog createGrokThreadWatchdog(Processor.Parameters parameters) { long intervalMillis = WATCHDOG_INTERVAL.get(parameters.env.settings()).getMillis(); long maxExecutionTimeMillis = WATCHDOG_MAX_EXECUTION_TIME.get(parameters.env.settings()).getMillis(); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java index 3b9e2121c9511..2735cf55776b0 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameFactoryTests.java @@ -20,18 +20,20 @@ package org.elasticsearch.ingest.common; import org.elasticsearch.ElasticsearchParseException; +import org.elasticsearch.ingest.TestTemplateService; import org.elasticsearch.test.ESTestCase; import org.hamcrest.Matchers; import org.joda.time.DateTimeZone; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.Map; public class DateIndexNameFactoryTests extends ESTestCase { public void testDefaults() throws Exception { - DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory(); + DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory(TestTemplateService.instance()); Map config = new HashMap<>(); config.put("field", "_field"); config.put("date_rounding", "y"); @@ -39,14 +41,14 @@ public void testDefaults() throws Exception { DateIndexNameProcessor processor = factory.create(null, null, config); assertThat(processor.getDateFormats().size(), Matchers.equalTo(1)); assertThat(processor.getField(), Matchers.equalTo("_field")); - assertThat(processor.getIndexNamePrefix(), Matchers.equalTo("")); - assertThat(processor.getDateRounding(), Matchers.equalTo("y")); - assertThat(processor.getIndexNameFormat(), Matchers.equalTo("yyyy-MM-dd")); + assertThat(processor.getIndexNamePrefixTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("")); + assertThat(processor.getDateRoundingTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("y")); + assertThat(processor.getIndexNameFormatTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("yyyy-MM-dd")); assertThat(processor.getTimezone(), Matchers.equalTo(DateTimeZone.UTC)); } public void testSpecifyOptionalSettings() throws Exception { - DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory(); + DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory(TestTemplateService.instance()); Map config = new HashMap<>(); config.put("field", "_field"); config.put("index_name_prefix", "_prefix"); @@ -63,7 +65,7 @@ public void testSpecifyOptionalSettings() throws Exception { config.put("index_name_format", "yyyyMMdd"); processor = factory.create(null, null, config); - assertThat(processor.getIndexNameFormat(), Matchers.equalTo("yyyyMMdd")); + assertThat(processor.getIndexNameFormatTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("yyyyMMdd")); config = new HashMap<>(); config.put("field", "_field"); @@ -80,11 +82,11 @@ public void testSpecifyOptionalSettings() throws Exception { config.put("date_rounding", "y"); processor = factory.create(null, null, config); - assertThat(processor.getIndexNamePrefix(), Matchers.equalTo("_prefix")); + assertThat(processor.getIndexNamePrefixTemplate().newInstance(Collections.emptyMap()).execute(), Matchers.equalTo("_prefix")); } public void testRequiredFields() throws Exception { - DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory(); + DateIndexNameProcessor.Factory factory = new DateIndexNameProcessor.Factory(TestTemplateService.instance()); Map config = new HashMap<>(); config.put("date_rounding", "y"); ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config)); @@ -95,5 +97,4 @@ public void testRequiredFields() throws Exception { e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config)); assertThat(e.getMessage(), Matchers.equalTo("[date_rounding] required property is missing")); } - } diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java index d052ce0cd44c3..52dba0f4e008b 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/DateIndexNameProcessorTests.java @@ -19,11 +19,14 @@ package org.elasticsearch.ingest.common; import org.elasticsearch.ingest.IngestDocument; +import org.elasticsearch.ingest.TestTemplateService; import org.elasticsearch.test.ESTestCase; import org.joda.time.DateTime; import org.joda.time.DateTimeZone; +import org.joda.time.format.DateTimeFormat; import java.util.Collections; +import java.util.List; import java.util.Locale; import java.util.function.Function; @@ -33,11 +36,8 @@ public class DateIndexNameProcessorTests extends ESTestCase { public void testJodaPattern() throws Exception { Function function = DateFormat.Joda.getFunction("yyyy-MM-dd'T'HH:mm:ss.SSSZ", DateTimeZone.UTC, Locale.ROOT); - DateIndexNameProcessor processor = new DateIndexNameProcessor( - "_tag", "_field", Collections.singletonList(function), DateTimeZone.UTC, - "events-", "y", "yyyyMMdd" - ); - + DateIndexNameProcessor processor = createProcessor("_field", Collections.singletonList(function), + DateTimeZone.UTC, "events-", "y", "yyyyMMdd"); IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null, null, Collections.singletonMap("_field", "2016-04-25T12:24:20.101Z")); processor.execute(document); @@ -46,7 +46,7 @@ public void testJodaPattern() throws Exception { public void testTAI64N()throws Exception { Function function = DateFormat.Tai64n.getFunction(null, DateTimeZone.UTC, null); - DateIndexNameProcessor dateProcessor = new DateIndexNameProcessor("_tag", "_field", Collections.singletonList(function), + DateIndexNameProcessor dateProcessor = createProcessor("_field", Collections.singletonList(function), DateTimeZone.UTC, "events-", "m", "yyyyMMdd"); IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null, null, Collections.singletonMap("_field", (randomBoolean() ? "@" : "") + "4000000050d506482dbdf024")); @@ -56,7 +56,7 @@ public void testTAI64N()throws Exception { public void testUnixMs()throws Exception { Function function = DateFormat.UnixMs.getFunction(null, DateTimeZone.UTC, null); - DateIndexNameProcessor dateProcessor = new DateIndexNameProcessor("_tag", "_field", Collections.singletonList(function), + DateIndexNameProcessor dateProcessor = createProcessor("_field", Collections.singletonList(function), DateTimeZone.UTC, "events-", "m", "yyyyMMdd"); IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null, null, Collections.singletonMap("_field", "1000500")); @@ -71,7 +71,7 @@ public void testUnixMs()throws Exception { public void testUnix()throws Exception { Function function = DateFormat.Unix.getFunction(null, DateTimeZone.UTC, null); - DateIndexNameProcessor dateProcessor = new DateIndexNameProcessor("_tag", "_field", Collections.singletonList(function), + DateIndexNameProcessor dateProcessor = createProcessor("_field", Collections.singletonList(function), DateTimeZone.UTC, "events-", "m", "yyyyMMdd"); IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null, null, Collections.singletonMap("_field", "1000.5")); @@ -79,4 +79,33 @@ public void testUnix()throws Exception { assertThat(document.getSourceAndMetadata().get("_index"), equalTo("")); } + public void testTemplatedFields() throws Exception { + String indexNamePrefix = randomAlphaOfLength(10); + String dateRounding = randomFrom("y", "M", "w", "d", "h", "m", "s"); + String indexNameFormat = randomFrom("yyyy-MM-dd'T'HH:mm:ss.SSSZ", "yyyyMMdd", "MM/dd/yyyy"); + String date = Integer.toString(randomInt()); + Function dateTimeFunction = DateFormat.Unix.getFunction(null, DateTimeZone.UTC, null); + + DateIndexNameProcessor dateProcessor = createProcessor("_field", + Collections.singletonList(dateTimeFunction), DateTimeZone.UTC, indexNamePrefix, + dateRounding, indexNameFormat); + + IngestDocument document = new IngestDocument("_index", "_type", "_id", null, null, null, null, + Collections.singletonMap("_field", date)); + dateProcessor.execute(document); + + assertThat(document.getSourceAndMetadata().get("_index"), + equalTo("<"+indexNamePrefix+"{"+DateTimeFormat.forPattern(indexNameFormat) + .print(dateTimeFunction.apply(date))+"||/"+dateRounding+"{"+indexNameFormat+"|UTC}}>")); + } + + private DateIndexNameProcessor createProcessor(String field, List> dateFormats, + DateTimeZone timezone, String indexNamePrefix, String dateRounding, + String indexNameFormat) { + return new DateIndexNameProcessor(randomAlphaOfLength(10), field, dateFormats, timezone, + new TestTemplateService.MockTemplateScript.Factory(indexNamePrefix), + new TestTemplateService.MockTemplateScript.Factory(dateRounding), + new TestTemplateService.MockTemplateScript.Factory(indexNameFormat) + ); + } } From 3de9494d6141b0d8be039de12ee6a0c32b174d2b Mon Sep 17 00:00:00 2001 From: Lisa Cawley Date: Wed, 11 Jul 2018 09:08:32 -0700 Subject: [PATCH 28/50] [DOCS] Removes alternative docker pull example (#31934) --- docs/reference/setup/install/docker.asciidoc | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/docs/reference/setup/install/docker.asciidoc b/docs/reference/setup/install/docker.asciidoc index 11ae7dbb98efb..d13f0bae1308a 100644 --- a/docs/reference/setup/install/docker.asciidoc +++ b/docs/reference/setup/install/docker.asciidoc @@ -7,8 +7,6 @@ The images use https://hub.docker.com/_/centos/[centos:7] as the base image. A list of all published Docker images and tags can be found in https://www.docker.elastic.co[www.docker.elastic.co]. The source code can be found on https://github.com/elastic/elasticsearch-docker/tree/{branch}[GitHub]. -==== Image types - These images are free to use under the Elastic license. They contain open source and free commercial features and access to paid commercial features. {xpack-ref}/license-management.html[Start a 30-day trial] to try out all of the @@ -16,9 +14,6 @@ paid commercial features. See the https://www.elastic.co/subscriptions[Subscriptions] page for information about Elastic license levels. -Alternatively, you can download `-oss` images, which contain only features that -are available under the Apache 2.0 license. - ==== Pulling the image Obtaining {es} for Docker is as simple as issuing a +docker pull+ command @@ -33,14 +28,17 @@ endif::[] ifeval::["{release-state}"!="unreleased"] -Docker images can be retrieved with the following commands: +For example, the Docker image can be retrieved with the following command: ["source","sh",subs="attributes"] -------------------------------------------- docker pull {docker-repo}:{version} -docker pull {docker-repo}-oss:{version} -------------------------------------------- +Alternatively, you can download other Docker images that contain only features +that are available under the Apache 2.0 license from +https://www.docker.elastic.co[www.docker.elastic.co]. + endif::[] [[docker-cli-run]] From e0c3b4551b6a361a3d285e4167a147c205c5fb62 Mon Sep 17 00:00:00 2001 From: Clinton Gormley Date: Wed, 11 Jul 2018 20:17:05 +0200 Subject: [PATCH 29/50] Docs: Added note about cloud service to installation and getting started --- docs/reference/getting-started.asciidoc | 5 +++++ docs/reference/setup/install.asciidoc | 7 ++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/reference/getting-started.asciidoc b/docs/reference/getting-started.asciidoc index b0889a723f511..4e2156093d61c 100755 --- a/docs/reference/getting-started.asciidoc +++ b/docs/reference/getting-started.asciidoc @@ -104,6 +104,11 @@ With that out of the way, let's get started with the fun part... == Installation +You can skip installation completely by using our hosted +Elasticsearch Service on https://www.elastic.co/cloud[Elastic Cloud], which is +available on AWS and GCP. You can +https://www.elastic.co/cloud/elasticsearch-service/signup[try out the hosted service] for free. + Elasticsearch requires at least Java 8. Specifically as of this writing, it is recommended that you use the Oracle JDK version {jdk}. Java installation varies from platform to platform so we won't go into those details here. Oracle's recommended installation documentation can be found on http://docs.oracle.com/javase/8/docs/technotes/guides/install/install_overview.html[Oracle's website]. Suffice to say, before you install Elasticsearch, please check your Java version first by running (and then install/upgrade accordingly if needed): [source,sh] diff --git a/docs/reference/setup/install.asciidoc b/docs/reference/setup/install.asciidoc index 783cb804e7af1..7675e5ad14659 100644 --- a/docs/reference/setup/install.asciidoc +++ b/docs/reference/setup/install.asciidoc @@ -1,6 +1,11 @@ [[install-elasticsearch]] == Installing Elasticsearch +Elasticsearch can be run on your own hardware or using our hosted +Elasticsearch Service on https://www.elastic.co/cloud[Elastic Cloud], which is +available on AWS and GCP. You can +https://www.elastic.co/cloud/elasticsearch-service/signup[try out the hosted service] for free. + Elasticsearch is provided in the following package formats: [horizontal] @@ -38,7 +43,7 @@ Elasticsearch on Windows. MSIs may be downloaded from the Elasticsearch website. `docker`:: Images are available for running Elasticsearch as Docker containers. They may be -downloaded from the Elastic Docker Registry. +downloaded from the Elastic Docker Registry. + {ref}/docker.html[Install {es} with Docker] From 907a0fed9c52483e75eb094de79c6d886b4f03e8 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 11 Jul 2018 10:04:17 -0400 Subject: [PATCH 30/50] Switch test framework to new style requests (#31939) In #29623 we added `Request` object flavored requests to the low level REST client and in #30315 we deprecated the old `performRequest`s. This changes all calls in the `test/framework` project to use the new versions. --- .../test/rest/ESRestTestCase.java | 86 +++++++++---------- .../rest/yaml/ESClientYamlSuiteTestCase.java | 8 +- 2 files changed, 44 insertions(+), 50 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java index 81a9598496bf5..937adddf3a43d 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java @@ -21,12 +21,6 @@ import org.apache.http.Header; import org.apache.http.HttpHost; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpHead; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpPut; -import org.apache.http.entity.ContentType; -import org.apache.http.entity.StringEntity; import org.apache.http.message.BasicHeader; import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy; import org.apache.http.ssl.SSLContexts; @@ -68,16 +62,12 @@ import java.security.NoSuchAlgorithmException; import java.security.cert.CertificateException; import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; -import static java.util.Collections.emptyMap; -import static java.util.Collections.singletonMap; import static java.util.Collections.sort; import static java.util.Collections.unmodifiableList; import static org.hamcrest.Matchers.anyOf; @@ -307,25 +297,25 @@ private void wipeCluster() throws IOException { * the snapshots intact in the repository. */ private void wipeSnapshots() throws IOException { - for (Map.Entry repo : entityAsMap(adminClient.performRequest("GET", "_snapshot/_all")).entrySet()) { + for (Map.Entry repo : entityAsMap(adminClient.performRequest(new Request("GET", "/_snapshot/_all"))).entrySet()) { String repoName = repo.getKey(); Map repoSpec = (Map) repo.getValue(); String repoType = (String) repoSpec.get("type"); if (false == preserveSnapshotsUponCompletion() && repoType.equals("fs")) { // All other repo types we really don't have a chance of being able to iterate properly, sadly. - String url = "_snapshot/" + repoName + "/_all"; - Map params = singletonMap("ignore_unavailable", "true"); - List snapshots = (List) entityAsMap(adminClient.performRequest("GET", url, params)).get("snapshots"); + Request listRequest = new Request("GET", "/_snapshot/" + repoName + "/_all"); + listRequest.addParameter("ignore_unavailable", "true"); + List snapshots = (List) entityAsMap(adminClient.performRequest(listRequest)).get("snapshots"); for (Object snapshot : snapshots) { Map snapshotInfo = (Map) snapshot; String name = (String) snapshotInfo.get("snapshot"); logger.debug("wiping snapshot [{}/{}]", repoName, name); - adminClient().performRequest("DELETE", "_snapshot/" + repoName + "/" + name); + adminClient().performRequest(new Request("DELETE", "/_snapshot/" + repoName + "/" + name)); } } if (preserveReposUponCompletion() == false) { logger.debug("wiping snapshot repository [{}]", repoName); - adminClient().performRequest("DELETE", "_snapshot/" + repoName); + adminClient().performRequest(new Request("DELETE", "_snapshot/" + repoName)); } } } @@ -334,7 +324,7 @@ private void wipeSnapshots() throws IOException { * Remove any cluster settings. */ private void wipeClusterSettings() throws IOException { - Map getResponse = entityAsMap(adminClient().performRequest("GET", "/_cluster/settings")); + Map getResponse = entityAsMap(adminClient().performRequest(new Request("GET", "/_cluster/settings"))); boolean mustClear = false; XContentBuilder clearCommand = JsonXContent.contentBuilder(); @@ -355,8 +345,9 @@ private void wipeClusterSettings() throws IOException { clearCommand.endObject(); if (mustClear) { - adminClient().performRequest("PUT", "/_cluster/settings", emptyMap(), new StringEntity( - Strings.toString(clearCommand), ContentType.APPLICATION_JSON)); + Request request = new Request("PUT", "/_cluster/settings"); + request.setJsonEntity(Strings.toString(clearCommand)); + adminClient().performRequest(request); } } @@ -365,7 +356,7 @@ private void wipeClusterSettings() throws IOException { * other tests. */ private void logIfThereAreRunningTasks() throws InterruptedException, IOException { - Set runningTasks = runningTasks(adminClient().performRequest("GET", "_tasks")); + Set runningTasks = runningTasks(adminClient().performRequest(new Request("GET", "/_tasks"))); // Ignore the task list API - it doesn't count against us runningTasks.remove(ListTasksAction.NAME); runningTasks.remove(ListTasksAction.NAME + "[n]"); @@ -389,7 +380,7 @@ private void logIfThereAreRunningTasks() throws InterruptedException, IOExceptio private void waitForClusterStateUpdatesToFinish() throws Exception { assertBusy(() -> { try { - Response response = adminClient().performRequest("GET", "_cluster/pending_tasks"); + Response response = adminClient().performRequest(new Request("GET", "/_cluster/pending_tasks")); List tasks = (List) entityAsMap(response).get("tasks"); if (false == tasks.isEmpty()) { StringBuilder message = new StringBuilder("there are still running tasks:"); @@ -514,12 +505,12 @@ protected static void assertOK(Response response) { * @param index index to test for **/ protected static void ensureGreen(String index) throws IOException { - Map params = new HashMap<>(); - params.put("wait_for_status", "green"); - params.put("wait_for_no_relocating_shards", "true"); - params.put("timeout", "70s"); - params.put("level", "shards"); - assertOK(client().performRequest("GET", "_cluster/health/" + index, params)); + Request request = new Request("GET", "/_cluster/health/" + index); + request.addParameter("wait_for_status", "green"); + request.addParameter("wait_for_no_relocating_shards", "true"); + request.addParameter("timeout", "70s"); + request.addParameter("level", "shards"); + client().performRequest(request); } /** @@ -527,11 +518,11 @@ protected static void ensureGreen(String index) throws IOException { * in the cluster and doesn't require to know how many nodes/replica there are. */ protected static void ensureNoInitializingShards() throws IOException { - Map params = new HashMap<>(); - params.put("wait_for_no_initializing_shards", "true"); - params.put("timeout", "70s"); - params.put("level", "shards"); - assertOK(client().performRequest("GET", "_cluster/health/", params)); + Request request = new Request("GET", "/_cluster/health"); + request.addParameter("wait_for_no_initializing_shards", "true"); + request.addParameter("timeout", "70s"); + request.addParameter("level", "shards"); + client().performRequest(request); } protected static void createIndex(String name, Settings settings) throws IOException { @@ -539,9 +530,10 @@ protected static void createIndex(String name, Settings settings) throws IOExcep } protected static void createIndex(String name, Settings settings, String mapping) throws IOException { - assertOK(client().performRequest(HttpPut.METHOD_NAME, name, Collections.emptyMap(), - new StringEntity("{ \"settings\": " + Strings.toString(settings) - + ", \"mappings\" : {" + mapping + "} }", ContentType.APPLICATION_JSON))); + Request request = new Request("PUT", "/" + name); + request.setJsonEntity("{\n \"settings\": " + Strings.toString(settings) + + ", \"mappings\" : {" + mapping + "} }"); + client().performRequest(request); } protected static void updateIndexSettings(String index, Settings.Builder settings) throws IOException { @@ -549,42 +541,42 @@ protected static void updateIndexSettings(String index, Settings.Builder setting } private static void updateIndexSettings(String index, Settings settings) throws IOException { - assertOK(client().performRequest("PUT", index + "/_settings", Collections.emptyMap(), - new StringEntity(Strings.toString(settings), ContentType.APPLICATION_JSON))); + Request request = new Request("PUT", "/" + index + "/_settings"); + request.setJsonEntity(Strings.toString(settings)); + client().performRequest(request); } protected static Map getIndexSettings(String index) throws IOException { - Map params = new HashMap<>(); - params.put("flat_settings", "true"); - Response response = client().performRequest(HttpGet.METHOD_NAME, index + "/_settings", params); - assertOK(response); + Request request = new Request("GET", "/" + index + "/_settings"); + request.addParameter("flat_settings", "true"); + Response response = client().performRequest(request); try (InputStream is = response.getEntity().getContent()) { return XContentHelper.convertToMap(XContentType.JSON.xContent(), is, true); } } protected static boolean indexExists(String index) throws IOException { - Response response = client().performRequest(HttpHead.METHOD_NAME, index); + Response response = client().performRequest(new Request("HEAD", "/" + index)); return RestStatus.OK.getStatus() == response.getStatusLine().getStatusCode(); } protected static void closeIndex(String index) throws IOException { - Response response = client().performRequest(HttpPost.METHOD_NAME, index + "/_close"); + Response response = client().performRequest(new Request("POST", "/" + index + "/_close")); assertThat(response.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus())); } protected static void openIndex(String index) throws IOException { - Response response = client().performRequest(HttpPost.METHOD_NAME, index + "/_open"); + Response response = client().performRequest(new Request("POST", "/" + index + "/_open")); assertThat(response.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus())); } protected static boolean aliasExists(String alias) throws IOException { - Response response = client().performRequest(HttpHead.METHOD_NAME, "/_alias/" + alias); + Response response = client().performRequest(new Request("HEAD", "/_alias/" + alias)); return RestStatus.OK.getStatus() == response.getStatusLine().getStatusCode(); } protected static boolean aliasExists(String index, String alias) throws IOException { - Response response = client().performRequest(HttpHead.METHOD_NAME, "/" + index + "/_alias/" + alias); + Response response = client().performRequest(new Request("HEAD", "/" + index + "/_alias/" + alias)); return RestStatus.OK.getStatus() == response.getStatusLine().getStatusCode(); } @@ -602,7 +594,7 @@ protected static Map getAlias(final String index, final String a } protected static Map getAsMap(final String endpoint) throws IOException { - Response response = client().performRequest(HttpGet.METHOD_NAME, endpoint); + Response response = client().performRequest(new Request("GET", endpoint)); XContentType entityContentType = XContentType.fromMediaTypeOrFormat(response.getEntity().getContentType().getValue()); Map responseEntity = XContentHelper.convertToMap(entityContentType.xContent(), response.getEntity().getContent(), false); diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java index 0082af64b9e94..8218ce5dddaca 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/rest/yaml/ESClientYamlSuiteTestCase.java @@ -23,6 +23,7 @@ import org.apache.http.HttpHost; import org.elasticsearch.Version; import org.elasticsearch.client.Node; +import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; @@ -46,7 +47,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; -import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; @@ -287,7 +287,9 @@ private static void validateSpec(ClientYamlSuiteRestSpec restSpec) { private static Tuple readVersionsFromCatNodes(RestClient restClient) throws IOException { // we simply go to the _cat/nodes API and parse all versions in the cluster - Response response = restClient.performRequest("GET", "/_cat/nodes", Collections.singletonMap("h", "version,master")); + Request request = new Request("GET", "/_cat/nodes"); + request.addParameter("h", "version,master"); + Response response = restClient.performRequest(request); ClientYamlTestResponse restTestResponse = new ClientYamlTestResponse(response); String nodesCatResponse = restTestResponse.getBodyAsString(); String[] split = nodesCatResponse.split("\n"); @@ -315,7 +317,7 @@ private static Version readVersionsFromInfo(RestClient restClient, int numHosts) Version version = null; for (int i = 0; i < numHosts; i++) { //we don't really use the urls here, we rely on the client doing round-robin to touch all the nodes in the cluster - Response response = restClient.performRequest("GET", "/"); + Response response = restClient.performRequest(new Request("GET", "/")); ClientYamlTestResponse restTestResponse = new ClientYamlTestResponse(response); Object latestVersion = restTestResponse.evaluate("version.number"); if (latestVersion == null) { From d3d7309dca15716239c4330aaecf675057f1b7eb Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 11 Jul 2018 14:42:55 -0400 Subject: [PATCH 31/50] Switch reindex tests to new style requests (#31941) In #29623 we added `Request` object flavored requests to the low level REST client and in #30315 we deprecated the old `performRequest`s. This changes all calls in the `modules/reindex` project to use the new versions. --- .../index/reindex/ManyDocumentsIT.java | 65 ++++++++++------- .../remote/ReindexFromOldRemoteIT.java | 72 ++++++++----------- 2 files changed, 70 insertions(+), 67 deletions(-) diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ManyDocumentsIT.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ManyDocumentsIT.java index e9082c96fd163..6aa1046492ccc 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ManyDocumentsIT.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/ManyDocumentsIT.java @@ -19,19 +19,13 @@ package org.elasticsearch.index.reindex; -import org.apache.http.entity.ContentType; -import org.apache.http.entity.StringEntity; -import org.elasticsearch.client.Response; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.client.Request; import org.elasticsearch.test.rest.ESRestTestCase; import org.junit.Before; import java.io.IOException; import java.util.Map; -import static java.util.Collections.emptyMap; -import static java.util.Collections.singletonMap; import static org.hamcrest.Matchers.hasEntry; /** @@ -50,48 +44,69 @@ public void setupTestIndex() throws IOException { bulk.append("{\"index\":{}}\n"); bulk.append("{\"test\":\"test\"}\n"); } - client().performRequest("POST", "/test/test/_bulk", singletonMap("refresh", "true"), - new StringEntity(bulk.toString(), ContentType.APPLICATION_JSON)); + Request request = new Request("POST", "/test/test/_bulk"); + request.addParameter("refresh", "true"); + request.setJsonEntity(bulk.toString()); + client().performRequest(request); } public void testReindex() throws IOException { - Map response = toMap(client().performRequest("POST", "/_reindex", emptyMap(), new StringEntity( - "{\"source\":{\"index\":\"test\"}, \"dest\":{\"index\":\"des\"}}", - ContentType.APPLICATION_JSON))); + Request request = new Request("POST", "/_reindex"); + request.setJsonEntity( + "{\n" + + " \"source\":{\n" + + " \"index\":\"test\"\n" + + " },\n" + + " \"dest\":{\n" + + " \"index\":\"des\"\n" + + " }\n" + + "}"); + Map response = entityAsMap(client().performRequest(request)); assertThat(response, hasEntry("total", count)); assertThat(response, hasEntry("created", count)); } public void testReindexFromRemote() throws IOException { - Map nodesInfo = toMap(client().performRequest("GET", "/_nodes/http")); + Map nodesInfo = entityAsMap(client().performRequest(new Request("GET", "/_nodes/http"))); nodesInfo = (Map) nodesInfo.get("nodes"); Map nodeInfo = (Map) nodesInfo.values().iterator().next(); Map http = (Map) nodeInfo.get("http"); String remote = "http://"+ http.get("publish_address"); - Map response = toMap(client().performRequest("POST", "/_reindex", emptyMap(), new StringEntity( - "{\"source\":{\"index\":\"test\",\"remote\":{\"host\":\"" + remote + "\"}}, \"dest\":{\"index\":\"des\"}}", - ContentType.APPLICATION_JSON))); + Request request = new Request("POST", "/_reindex"); + request.setJsonEntity( + "{\n" + + " \"source\":{\n" + + " \"index\":\"test\",\n" + + " \"remote\":{\n" + + " \"host\":\"" + remote + "\"\n" + + " }\n" + + " }\n," + + " \"dest\":{\n" + + " \"index\":\"des\"\n" + + " }\n" + + "}"); + Map response = entityAsMap(client().performRequest(request)); assertThat(response, hasEntry("total", count)); assertThat(response, hasEntry("created", count)); } public void testUpdateByQuery() throws IOException { - Map response = toMap(client().performRequest("POST", "/test/_update_by_query")); + Map response = entityAsMap(client().performRequest(new Request("POST", "/test/_update_by_query"))); assertThat(response, hasEntry("total", count)); assertThat(response, hasEntry("updated", count)); } public void testDeleteByQuery() throws IOException { - Map response = toMap(client().performRequest("POST", "/test/_delete_by_query", emptyMap(), new StringEntity( - "{\"query\":{\"match_all\":{}}}", - ContentType.APPLICATION_JSON))); + Request request = new Request("POST", "/test/_delete_by_query"); + request.setJsonEntity( + "{\n" + + " \"query\":{\n" + + " \"match_all\": {}\n" + + " }\n" + + "}"); + Map response = entityAsMap(client().performRequest(request)); assertThat(response, hasEntry("total", count)); assertThat(response, hasEntry("deleted", count)); } - - static Map toMap(Response response) throws IOException { - return XContentHelper.convertToMap(JsonXContent.jsonXContent, response.getEntity().getContent(), false); - } - } diff --git a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/ReindexFromOldRemoteIT.java b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/ReindexFromOldRemoteIT.java index 5d359053a6668..9feed83595ff1 100644 --- a/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/ReindexFromOldRemoteIT.java +++ b/modules/reindex/src/test/java/org/elasticsearch/index/reindex/remote/ReindexFromOldRemoteIT.java @@ -19,25 +19,24 @@ package org.elasticsearch.index.reindex.remote; -import org.apache.http.HttpEntity; import org.apache.http.HttpHost; -import org.apache.http.entity.ContentType; -import org.apache.http.entity.StringEntity; import org.apache.http.util.EntityUtils; +import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; -import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; import org.elasticsearch.common.Booleans; import org.elasticsearch.test.rest.ESRestTestCase; import java.io.IOException; -import java.util.Map; -import java.util.TreeMap; -import static java.util.Collections.singletonMap; import static org.hamcrest.Matchers.containsString; public class ReindexFromOldRemoteIT extends ESRestTestCase { + /** + * Number of documents to test when reindexing from an old version. + */ + private static final int DOCS = 5; + private void oldEsTestCase(String portPropertyName, String requestsPerSecond) throws IOException { boolean enabled = Booleans.parseBoolean(System.getProperty("tests.fromOld")); assumeTrue("test is disabled, probably because this is windows", enabled); @@ -45,17 +44,19 @@ private void oldEsTestCase(String portPropertyName, String requestsPerSecond) th int oldEsPort = Integer.parseInt(System.getProperty(portPropertyName)); try (RestClient oldEs = RestClient.builder(new HttpHost("127.0.0.1", oldEsPort)).build()) { try { - HttpEntity entity = new StringEntity("{\"settings\":{\"number_of_shards\": 1}}", ContentType.APPLICATION_JSON); - oldEs.performRequest("PUT", "/test", singletonMap("refresh", "true"), entity); - - entity = new StringEntity("{\"test\":\"test\"}", ContentType.APPLICATION_JSON); - oldEs.performRequest("PUT", "/test/doc/testdoc1", singletonMap("refresh", "true"), entity); - oldEs.performRequest("PUT", "/test/doc/testdoc2", singletonMap("refresh", "true"), entity); - oldEs.performRequest("PUT", "/test/doc/testdoc3", singletonMap("refresh", "true"), entity); - oldEs.performRequest("PUT", "/test/doc/testdoc4", singletonMap("refresh", "true"), entity); - oldEs.performRequest("PUT", "/test/doc/testdoc5", singletonMap("refresh", "true"), entity); + Request createIndex = new Request("PUT", "/test"); + createIndex.setJsonEntity("{\"settings\":{\"number_of_shards\": 1}}"); + oldEs.performRequest(createIndex); + + for (int i = 0; i < DOCS; i++) { + Request doc = new Request("PUT", "/test/doc/testdoc" + i); + doc.addParameter("refresh", "true"); + doc.setJsonEntity("{\"test\":\"test\"}"); + oldEs.performRequest(doc); + } - entity = new StringEntity( + Request reindex = new Request("POST", "/_reindex"); + reindex.setJsonEntity( "{\n" + " \"source\":{\n" + " \"index\": \"test\",\n" @@ -67,36 +68,23 @@ private void oldEsTestCase(String portPropertyName, String requestsPerSecond) th + " \"dest\": {\n" + " \"index\": \"test\"\n" + " }\n" - + "}", - ContentType.APPLICATION_JSON); - Map params = new TreeMap<>(); - params.put("refresh", "true"); - params.put("pretty", "true"); + + "}"); + reindex.addParameter("refresh", "true"); + reindex.addParameter("pretty", "true"); if (requestsPerSecond != null) { - params.put("requests_per_second", requestsPerSecond); + reindex.addParameter("requests_per_second", requestsPerSecond); } - client().performRequest("POST", "/_reindex", params, entity); + client().performRequest(reindex); - Response response = client().performRequest("POST", "test/_search", singletonMap("pretty", "true")); + Request search = new Request("POST", "/test/_search"); + search.addParameter("pretty", "true"); + Response response = client().performRequest(search); String result = EntityUtils.toString(response.getEntity()); - assertThat(result, containsString("\"_id\" : \"testdoc1\"")); - } finally { - try { - oldEs.performRequest("DELETE", "/test"); - } catch (ResponseException e) { - /* Try not to throw ResponseException for as it'll eat the - * real exception. This is because the rest client throws - * exceptions in a "funny" way that isn't compatible with - * `suppressed`. In the case of 404s we'll just log something - * and move on because that just means that a previous - * failure caused the index not to be created. */ - if (e.getResponse().getStatusLine().getStatusCode() == 404) { - logger.warn("old index not deleted because it doesn't exist"); - } else { - logger.error("failed to remove old index", e); - fail("failed to remove old index, see log"); - } + for (int i = 0; i < DOCS; i++) { + assertThat(result, containsString("\"_id\" : \"testdoc" + i + "\"")); } + } finally { + oldEs.performRequest(new Request("DELETE", "/test")); } } } From d37ac56982d94dc82fdb799e3a422d8a8ece28da Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 11 Jul 2018 14:52:45 -0400 Subject: [PATCH 32/50] Switch url repository rest tests to new style requests (#31944) In #29623 we added `Request` object flavored requests to the low level REST client and in #30315 we deprecated the old `performRequest`s. This changes all calls in the `module/repository-url` project to use the new versions. --- .../RepositoryURLClientYamlTestSuiteIT.java | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java index f33fa98f0e3be..65d9b87b07d4c 100644 --- a/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java +++ b/modules/repository-url/src/test/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java @@ -24,6 +24,7 @@ import org.apache.http.HttpEntity; import org.apache.http.entity.ContentType; import org.apache.http.nio.entity.NStringEntity; +import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.PathUtils; @@ -44,7 +45,6 @@ import java.util.List; import java.util.Map; -import static java.util.Collections.emptyMap; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasSize; @@ -70,8 +70,10 @@ public static Iterable parameters() throws Exception { **/ @Before public void registerRepositories() throws IOException { - Response clusterSettingsResponse = client().performRequest("GET", "/_cluster/settings?include_defaults=true" + - "&filter_path=defaults.path.repo,defaults.repositories.url.allowed_urls"); + Request clusterSettingsRequest = new Request("GET", "/_cluster/settings"); + clusterSettingsRequest.addParameter("include_defaults", "true"); + clusterSettingsRequest.addParameter("filter_path", "defaults.path.repo,defaults.repositories.url.allowed_urls"); + Response clusterSettingsResponse = client().performRequest(clusterSettingsRequest); Map clusterSettings = entityAsMap(clusterSettingsResponse); @SuppressWarnings("unchecked") @@ -83,13 +85,17 @@ public void registerRepositories() throws IOException { final URI pathRepoUri = PathUtils.get(pathRepo).toUri().normalize(); // Create a FS repository using the path.repo location - Response createFsRepositoryResponse = client().performRequest("PUT", "_snapshot/repository-fs", emptyMap(), - buildRepositorySettings(FsRepository.TYPE, Settings.builder().put("location", pathRepo).build())); + Request createFsRepositoryRequest = new Request("PUT", "/_snapshot/repository-fs"); + createFsRepositoryRequest.setEntity(buildRepositorySettings(FsRepository.TYPE, + Settings.builder().put("location", pathRepo).build())); + Response createFsRepositoryResponse = client().performRequest(createFsRepositoryRequest); assertThat(createFsRepositoryResponse.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus())); // Create a URL repository using the file://{path.repo} URL - Response createFileRepositoryResponse = client().performRequest("PUT", "_snapshot/repository-file", emptyMap(), - buildRepositorySettings(URLRepository.TYPE, Settings.builder().put("url", pathRepoUri.toString()).build())); + Request createFileRepositoryRequest = new Request("PUT", "/_snapshot/repository-file"); + createFileRepositoryRequest.setEntity(buildRepositorySettings(URLRepository.TYPE, + Settings.builder().put("url", pathRepoUri.toString()).build())); + Response createFileRepositoryResponse = client().performRequest(createFileRepositoryRequest); assertThat(createFileRepositoryResponse.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus())); // Create a URL repository using the http://{fixture} URL @@ -99,8 +105,10 @@ public void registerRepositories() throws IOException { try { InetAddress inetAddress = InetAddress.getByName(new URL(allowedUrl).getHost()); if (inetAddress.isAnyLocalAddress() || inetAddress.isLoopbackAddress()) { - Response createUrlRepositoryResponse = client().performRequest("PUT", "_snapshot/repository-url", emptyMap(), - buildRepositorySettings(URLRepository.TYPE, Settings.builder().put("url", allowedUrl).build())); + Request createUrlRepositoryRequest = new Request("PUT", "/_snapshot/repository-url"); + createUrlRepositoryRequest.setEntity(buildRepositorySettings(URLRepository.TYPE, + Settings.builder().put("url", allowedUrl).build())); + Response createUrlRepositoryResponse = client().performRequest(createUrlRepositoryRequest); assertThat(createUrlRepositoryResponse.getStatusLine().getStatusCode(), equalTo(RestStatus.OK.getStatus())); break; } @@ -126,4 +134,3 @@ private static HttpEntity buildRepositorySettings(final String type, final Setti } } } - From d385dcd0cd92958ca39c2e5725e0226be95e8e02 Mon Sep 17 00:00:00 2001 From: Jimi Ford Date: Wed, 11 Jul 2018 15:01:49 -0400 Subject: [PATCH 33/50] Docs: fix typo in datehistogram (#31972) --- .../aggregations/bucket/datehistogram-aggregation.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc b/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc index c2d1614ad6e56..efbd8ef7389bb 100644 --- a/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc +++ b/docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc @@ -33,7 +33,7 @@ Available expressions for interval: `year` (`1y`), `quarter` (`1q`), `month` (`1 Time values can also be specified via abbreviations supported by <> parsing. Note that fractional time values are not supported, but you can address this by shifting to another time unit (e.g., `1.5h` could instead be specified as `90m`). Also note that time intervals larger than -than days do not support arbitrary values but can only be one unit large (e.g. `1y` is valid, `2y` is not). +days do not support arbitrary values but can only be one unit large (e.g. `1y` is valid, `2y` is not). [source,js] -------------------------------------------------- From 5440fb4cc91b5a0875d70264d93328e8727655a7 Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Wed, 11 Jul 2018 23:31:46 +0300 Subject: [PATCH 34/50] SQL: HAVING clause should accept only aggregates (#31872) Improve Verifier to allow HAVING clauses only on aggregates Close #31726 (cherry picked from commit 6136e49a05f370376341d55d84ab0cd895f38a34) --- .../xpack/sql/analysis/analyzer/Verifier.java | 65 +++++++++++++++++-- .../analyzer/VerifierErrorMessagesTests.java | 10 +++ 2 files changed, 69 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java index 6f8be61b463fd..4915a25a55bc7 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/analyzer/Verifier.java @@ -213,10 +213,11 @@ static Collection verify(LogicalPlan plan) { * Check validity of Aggregate/GroupBy. * This rule is needed for multiple reasons: * 1. a user might specify an invalid aggregate (SELECT foo GROUP BY bar) - * 2. the order/having might contain a non-grouped attribute. This is typically + * 2. the ORDER BY/HAVING might contain a non-grouped attribute. This is typically * caught by the Analyzer however if wrapped in a function (ABS()) it gets resolved * (because the expression gets resolved little by little without being pushed down, * without the Analyzer modifying anything. + * 2a. HAVING also requires an Aggregate function * 3. composite agg (used for GROUP BY) allows ordering only on the group keys */ private static boolean checkGroupBy(LogicalPlan p, Set localFailures, @@ -244,7 +245,7 @@ private static boolean checkGroupByOrder(LogicalPlan p, Set localFailur } // make sure to compare attributes directly - if (Expressions.anyMatch(a.groupings(), + if (Expressions.anyMatch(a.groupings(), g -> e.semanticEquals(e instanceof Attribute ? Expressions.attribute(g) : g))) { return; } @@ -278,13 +279,14 @@ private static boolean checkGroupByHaving(LogicalPlan p, Set localFailu Map> missing = new LinkedHashMap<>(); Expression condition = f.condition(); - condition.collectFirstChildren(c -> checkGroupMatch(c, condition, a.groupings(), missing, functions)); + // variation of checkGroupMatch customized for HAVING, which requires just aggregations + condition.collectFirstChildren(c -> checkGroupByHavingHasOnlyAggs(c, condition, missing, functions)); if (!missing.isEmpty()) { String plural = missing.size() > 1 ? "s" : StringUtils.EMPTY; - localFailures.add(fail(condition, "Cannot filter by non-grouped column" + plural + " %s, expected %s", - Expressions.names(missing.keySet()), - Expressions.names(a.groupings()))); + localFailures.add( + fail(condition, "Cannot filter HAVING on non-aggregate" + plural + " %s; consider using WHERE instead", + Expressions.names(missing.keySet()))); groupingFailures.add(a); return false; } @@ -294,6 +296,57 @@ private static boolean checkGroupByHaving(LogicalPlan p, Set localFailu } + private static boolean checkGroupByHavingHasOnlyAggs(Expression e, Node source, + Map> missing, Map functions) { + + // resolve FunctionAttribute to backing functions + if (e instanceof FunctionAttribute) { + FunctionAttribute fa = (FunctionAttribute) e; + Function function = functions.get(fa.functionId()); + // TODO: this should be handled by a different rule + if (function == null) { + return false; + } + e = function; + } + + // scalar functions can be a binary tree + // first test the function against the grouping + // and if that fails, start unpacking hoping to find matches + if (e instanceof ScalarFunction) { + ScalarFunction sf = (ScalarFunction) e; + + // unwrap function to find the base + for (Expression arg : sf.arguments()) { + arg.collectFirstChildren(c -> checkGroupByHavingHasOnlyAggs(c, source, missing, functions)); + } + return true; + + } else if (e instanceof Score) { + // Score can't be used for having + missing.put(e, source); + return true; + } + + // skip literals / foldable + if (e.foldable()) { + return true; + } + // skip aggs (allowed to refer to non-group columns) + if (Functions.isAggregate(e)) { + return true; + } + + // left without leaves which have to match; that's a failure since everything should be based on an agg + if (e instanceof Attribute) { + missing.put(e, source); + return true; + } + + return false; + } + + // check whether plain columns specified in an agg are mentioned in the group-by private static boolean checkGroupByAgg(LogicalPlan p, Set localFailures, Set groupingFailures, Map functions) { diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java index 60875e0194a0c..dce665a97e95d 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/analyzer/VerifierErrorMessagesTests.java @@ -159,4 +159,14 @@ public void testGroupByOrderByScore() { assertEquals("1:44: Cannot order by non-grouped column [SCORE()], expected [int]", verify("SELECT int FROM test GROUP BY int ORDER BY SCORE()")); } + + public void testHavingOnColumn() { + assertEquals("1:42: Cannot filter HAVING on non-aggregate [int]; consider using WHERE instead", + verify("SELECT int FROM test GROUP BY int HAVING int > 2")); + } + + public void testHavingOnScalar() { + assertEquals("1:42: Cannot filter HAVING on non-aggregate [int]; consider using WHERE instead", + verify("SELECT int FROM test GROUP BY int HAVING 2 < ABS(int)")); + } } \ No newline at end of file From 4214375a0c3424f5b8b4e3b48a54b9929841623d Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Wed, 11 Jul 2018 23:36:39 +0300 Subject: [PATCH 35/50] SQL: Support for escape sequences (#31884) Enhance grammar to allow JDBC/ODBC escape sequences, namely - date, time and timestamp {d ''}, {t ''} and {ts ''} - guid {guid ''} - LIKE escape {escape ''} - scalar function {fn } Fix #31883 (cherry picked from commit dc633e0000679da7ff650c9a11f53b8e17c0df96) --- x-pack/plugin/sql/src/main/antlr/SqlBase.g4 | 62 +- .../plugin/sql/src/main/antlr/SqlBase.tokens | 96 +- .../sql/src/main/antlr/SqlBaseLexer.tokens | 94 +- .../xpack/sql/parser/ExpressionBuilder.java | 167 +- .../xpack/sql/parser/LogicalPlanBuilder.java | 11 +- .../xpack/sql/parser/SqlBaseBaseListener.java | 153 +- .../xpack/sql/parser/SqlBaseBaseVisitor.java | 91 +- .../xpack/sql/parser/SqlBaseLexer.java | 598 ++-- .../xpack/sql/parser/SqlBaseListener.java | 141 +- .../xpack/sql/parser/SqlBaseParser.java | 2542 +++++++++++------ .../xpack/sql/parser/SqlBaseVisitor.java | 85 +- .../xpack/sql/parser/SqlParser.java | 19 +- .../sql/parser/EscapedFunctionsTests.java | 237 ++ 13 files changed, 2921 insertions(+), 1375 deletions(-) create mode 100644 x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/EscapedFunctionsTests.java diff --git a/x-pack/plugin/sql/src/main/antlr/SqlBase.g4 b/x-pack/plugin/sql/src/main/antlr/SqlBase.g4 index ea0b7da161c1e..2c3288babd6d8 100644 --- a/x-pack/plugin/sql/src/main/antlr/SqlBase.g4 +++ b/x-pack/plugin/sql/src/main/antlr/SqlBase.g4 @@ -74,9 +74,14 @@ queryNoWith : queryTerm /** we could add sort by - sort per partition */ (ORDER BY orderBy (',' orderBy)*)? - (LIMIT limit=(INTEGER_VALUE | ALL))? + limitClause? ; +limitClause + : LIMIT limit=(INTEGER_VALUE | ALL) + | LIMIT_ESC limit=(INTEGER_VALUE | ALL) ESC_END + ; + queryTerm : querySpecification #queryPrimaryDefault | '(' queryNoWith ')' #subquery @@ -185,7 +190,12 @@ predicate ; pattern - : value=string (ESCAPE escape=string)? + : value=string patternEscape? + ; + +patternEscape + : ESCAPE escape=string + | ESCAPE_ESC escape=string '}' ; valueExpression @@ -197,18 +207,44 @@ valueExpression ; primaryExpression - : CAST '(' expression AS dataType ')' #cast - | EXTRACT '(' field=identifier FROM valueExpression ')' #extract + : castExpression #cast + | extractExpression #extract | constant #constantDefault | ASTERISK #star | (qualifiedName DOT)? ASTERISK #star - | identifier '(' (setQuantifier? expression (',' expression)*)? ')' #functionCall + | functionExpression #function | '(' query ')' #subqueryExpression | identifier #columnReference | qualifiedName #dereference | '(' expression ')' #parenthesizedExpression ; +castExpression + : castTemplate + | FUNCTION_ESC castTemplate ESC_END + ; + +castTemplate + : CAST '(' expression AS dataType ')' + ; + +extractExpression + : extractTemplate + | FUNCTION_ESC extractTemplate ESC_END + ; + +extractTemplate + : EXTRACT '(' field=identifier FROM valueExpression ')' + ; + +functionExpression + : functionTemplate + | FUNCTION_ESC functionTemplate '}' + ; + +functionTemplate + : identifier '(' (setQuantifier? expression (',' expression)*)? ')' + ; constant : NULL #nullLiteral @@ -216,6 +252,10 @@ constant | booleanValue #booleanLiteral | STRING+ #stringLiteral | PARAM #paramLiteral + | DATE_ESC string ESC_END #dateEscapedLiteral + | TIME_ESC string ESC_END #timeEscapedLiteral + | TIMESTAMP_ESC string ESC_END #timestampEscapedLiteral + | GUID_ESC string ESC_END #guidEscapedLiteral ; comparisonOperator @@ -351,6 +391,18 @@ VERIFY: 'VERIFY'; WHERE: 'WHERE'; WITH: 'WITH'; +// Escaped Sequence +ESCAPE_ESC: '{ESCAPE'; +FUNCTION_ESC: '{FN'; +LIMIT_ESC:'{LIMIT'; +DATE_ESC: '{D'; +TIME_ESC: '{T'; +TIMESTAMP_ESC: '{TS'; +// mapped to string literal +GUID_ESC: '{GUID'; + +ESC_END: '}'; + EQ : '='; NEQ : '<>' | '!=' | '<=>'; LT : '<'; diff --git a/x-pack/plugin/sql/src/main/antlr/SqlBase.tokens b/x-pack/plugin/sql/src/main/antlr/SqlBase.tokens index 87cf9a4809d4a..527cc676e1d80 100644 --- a/x-pack/plugin/sql/src/main/antlr/SqlBase.tokens +++ b/x-pack/plugin/sql/src/main/antlr/SqlBase.tokens @@ -69,33 +69,41 @@ USING=68 VERIFY=69 WHERE=70 WITH=71 -EQ=72 -NEQ=73 -LT=74 -LTE=75 -GT=76 -GTE=77 -PLUS=78 -MINUS=79 -ASTERISK=80 -SLASH=81 -PERCENT=82 -CONCAT=83 -DOT=84 -PARAM=85 -STRING=86 -INTEGER_VALUE=87 -DECIMAL_VALUE=88 -IDENTIFIER=89 -DIGIT_IDENTIFIER=90 -TABLE_IDENTIFIER=91 -QUOTED_IDENTIFIER=92 -BACKQUOTED_IDENTIFIER=93 -SIMPLE_COMMENT=94 -BRACKETED_COMMENT=95 -WS=96 -UNRECOGNIZED=97 -DELIMITER=98 +ESCAPE_ESC=72 +FUNCTION_ESC=73 +LIMIT_ESC=74 +DATE_ESC=75 +TIME_ESC=76 +TIMESTAMP_ESC=77 +GUID_ESC=78 +ESC_END=79 +EQ=80 +NEQ=81 +LT=82 +LTE=83 +GT=84 +GTE=85 +PLUS=86 +MINUS=87 +ASTERISK=88 +SLASH=89 +PERCENT=90 +CONCAT=91 +DOT=92 +PARAM=93 +STRING=94 +INTEGER_VALUE=95 +DECIMAL_VALUE=96 +IDENTIFIER=97 +DIGIT_IDENTIFIER=98 +TABLE_IDENTIFIER=99 +QUOTED_IDENTIFIER=100 +BACKQUOTED_IDENTIFIER=101 +SIMPLE_COMMENT=102 +BRACKETED_COMMENT=103 +WS=104 +UNRECOGNIZED=105 +DELIMITER=106 '('=1 ')'=2 ','=3 @@ -167,16 +175,24 @@ DELIMITER=98 'VERIFY'=69 'WHERE'=70 'WITH'=71 -'='=72 -'<'=74 -'<='=75 -'>'=76 -'>='=77 -'+'=78 -'-'=79 -'*'=80 -'/'=81 -'%'=82 -'||'=83 -'.'=84 -'?'=85 +'{ESCAPE'=72 +'{FN'=73 +'{LIMIT'=74 +'{D'=75 +'{T'=76 +'{TS'=77 +'{GUID'=78 +'}'=79 +'='=80 +'<'=82 +'<='=83 +'>'=84 +'>='=85 +'+'=86 +'-'=87 +'*'=88 +'/'=89 +'%'=90 +'||'=91 +'.'=92 +'?'=93 diff --git a/x-pack/plugin/sql/src/main/antlr/SqlBaseLexer.tokens b/x-pack/plugin/sql/src/main/antlr/SqlBaseLexer.tokens index a687a9215ecb5..155d4860e0ea0 100644 --- a/x-pack/plugin/sql/src/main/antlr/SqlBaseLexer.tokens +++ b/x-pack/plugin/sql/src/main/antlr/SqlBaseLexer.tokens @@ -69,32 +69,40 @@ USING=68 VERIFY=69 WHERE=70 WITH=71 -EQ=72 -NEQ=73 -LT=74 -LTE=75 -GT=76 -GTE=77 -PLUS=78 -MINUS=79 -ASTERISK=80 -SLASH=81 -PERCENT=82 -CONCAT=83 -DOT=84 -PARAM=85 -STRING=86 -INTEGER_VALUE=87 -DECIMAL_VALUE=88 -IDENTIFIER=89 -DIGIT_IDENTIFIER=90 -TABLE_IDENTIFIER=91 -QUOTED_IDENTIFIER=92 -BACKQUOTED_IDENTIFIER=93 -SIMPLE_COMMENT=94 -BRACKETED_COMMENT=95 -WS=96 -UNRECOGNIZED=97 +ESCAPE_ESC=72 +FUNCTION_ESC=73 +LIMIT_ESC=74 +DATE_ESC=75 +TIME_ESC=76 +TIMESTAMP_ESC=77 +GUID_ESC=78 +ESC_END=79 +EQ=80 +NEQ=81 +LT=82 +LTE=83 +GT=84 +GTE=85 +PLUS=86 +MINUS=87 +ASTERISK=88 +SLASH=89 +PERCENT=90 +CONCAT=91 +DOT=92 +PARAM=93 +STRING=94 +INTEGER_VALUE=95 +DECIMAL_VALUE=96 +IDENTIFIER=97 +DIGIT_IDENTIFIER=98 +TABLE_IDENTIFIER=99 +QUOTED_IDENTIFIER=100 +BACKQUOTED_IDENTIFIER=101 +SIMPLE_COMMENT=102 +BRACKETED_COMMENT=103 +WS=104 +UNRECOGNIZED=105 '('=1 ')'=2 ','=3 @@ -166,16 +174,24 @@ UNRECOGNIZED=97 'VERIFY'=69 'WHERE'=70 'WITH'=71 -'='=72 -'<'=74 -'<='=75 -'>'=76 -'>='=77 -'+'=78 -'-'=79 -'*'=80 -'/'=81 -'%'=82 -'||'=83 -'.'=84 -'?'=85 +'{ESCAPE'=72 +'{FN'=73 +'{LIMIT'=74 +'{D'=75 +'{T'=76 +'{TS'=77 +'{GUID'=78 +'}'=79 +'='=80 +'<'=82 +'<='=83 +'>'=84 +'>='=85 +'+'=86 +'-'=87 +'*'=88 +'/'=89 +'%'=90 +'||'=91 +'.'=92 +'?'=93 diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java index 35eb76af67c7b..66ec98ea53c82 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/ExpressionBuilder.java @@ -20,6 +20,7 @@ import org.elasticsearch.xpack.sql.expression.ScalarSubquery; import org.elasticsearch.xpack.sql.expression.UnresolvedAttribute; import org.elasticsearch.xpack.sql.expression.UnresolvedStar; +import org.elasticsearch.xpack.sql.expression.function.Function; import org.elasticsearch.xpack.sql.expression.function.UnresolvedFunction; import org.elasticsearch.xpack.sql.expression.function.scalar.Cast; import org.elasticsearch.xpack.sql.expression.function.scalar.arithmetic.Add; @@ -48,14 +49,19 @@ import org.elasticsearch.xpack.sql.parser.SqlBaseParser.ArithmeticBinaryContext; import org.elasticsearch.xpack.sql.parser.SqlBaseParser.ArithmeticUnaryContext; import org.elasticsearch.xpack.sql.parser.SqlBaseParser.BooleanLiteralContext; -import org.elasticsearch.xpack.sql.parser.SqlBaseParser.CastContext; +import org.elasticsearch.xpack.sql.parser.SqlBaseParser.CastExpressionContext; +import org.elasticsearch.xpack.sql.parser.SqlBaseParser.CastTemplateContext; import org.elasticsearch.xpack.sql.parser.SqlBaseParser.ColumnReferenceContext; import org.elasticsearch.xpack.sql.parser.SqlBaseParser.ComparisonContext; +import org.elasticsearch.xpack.sql.parser.SqlBaseParser.DateEscapedLiteralContext; import org.elasticsearch.xpack.sql.parser.SqlBaseParser.DecimalLiteralContext; import org.elasticsearch.xpack.sql.parser.SqlBaseParser.DereferenceContext; import org.elasticsearch.xpack.sql.parser.SqlBaseParser.ExistsContext; -import org.elasticsearch.xpack.sql.parser.SqlBaseParser.ExtractContext; -import org.elasticsearch.xpack.sql.parser.SqlBaseParser.FunctionCallContext; +import org.elasticsearch.xpack.sql.parser.SqlBaseParser.ExtractExpressionContext; +import org.elasticsearch.xpack.sql.parser.SqlBaseParser.ExtractTemplateContext; +import org.elasticsearch.xpack.sql.parser.SqlBaseParser.FunctionExpressionContext; +import org.elasticsearch.xpack.sql.parser.SqlBaseParser.FunctionTemplateContext; +import org.elasticsearch.xpack.sql.parser.SqlBaseParser.GuidEscapedLiteralContext; import org.elasticsearch.xpack.sql.parser.SqlBaseParser.IntegerLiteralContext; import org.elasticsearch.xpack.sql.parser.SqlBaseParser.LogicalBinaryContext; import org.elasticsearch.xpack.sql.parser.SqlBaseParser.LogicalNotContext; @@ -66,6 +72,7 @@ import org.elasticsearch.xpack.sql.parser.SqlBaseParser.ParamLiteralContext; import org.elasticsearch.xpack.sql.parser.SqlBaseParser.ParenthesizedExpressionContext; import org.elasticsearch.xpack.sql.parser.SqlBaseParser.PatternContext; +import org.elasticsearch.xpack.sql.parser.SqlBaseParser.PatternEscapeContext; import org.elasticsearch.xpack.sql.parser.SqlBaseParser.PredicateContext; import org.elasticsearch.xpack.sql.parser.SqlBaseParser.PredicatedContext; import org.elasticsearch.xpack.sql.parser.SqlBaseParser.PrimitiveDataTypeContext; @@ -76,10 +83,16 @@ import org.elasticsearch.xpack.sql.parser.SqlBaseParser.StringLiteralContext; import org.elasticsearch.xpack.sql.parser.SqlBaseParser.StringQueryContext; import org.elasticsearch.xpack.sql.parser.SqlBaseParser.SubqueryExpressionContext; +import org.elasticsearch.xpack.sql.parser.SqlBaseParser.TimeEscapedLiteralContext; +import org.elasticsearch.xpack.sql.parser.SqlBaseParser.TimestampEscapedLiteralContext; import org.elasticsearch.xpack.sql.proto.SqlTypedParamValue; import org.elasticsearch.xpack.sql.tree.Location; import org.elasticsearch.xpack.sql.type.DataType; import org.elasticsearch.xpack.sql.type.DataTypes; +import org.joda.time.DateTime; +import org.joda.time.format.DateTimeFormatter; +import org.joda.time.format.DateTimeFormatterBuilder; +import org.joda.time.format.ISODateTimeFormat; import java.math.BigDecimal; import java.util.List; @@ -222,17 +235,18 @@ public LikePattern visitPattern(PatternContext ctx) { } char escape = 0; - String escapeString = string(ctx.escape); + PatternEscapeContext escapeCtx = ctx.patternEscape(); + String escapeString = escapeCtx == null ? null : string(escapeCtx.escape); if (Strings.hasText(escapeString)) { // shouldn't happen but adding validation in case the string parsing gets wonky if (escapeString.length() > 1) { - throw new ParsingException(source(ctx.escape), "A character not a string required for escaping; found [{}]", escapeString); + throw new ParsingException(source(escapeCtx), "A character not a string required for escaping; found [{}]", escapeString); } else if (escapeString.length() == 1) { escape = escapeString.charAt(0); // these chars already have a meaning if (escape == '*' || escape == '%' || escape == '_') { - throw new ParsingException(source(ctx.escape), "Char [{}] cannot be used for escaping", escape); + throw new ParsingException(source(escapeCtx.escape), "Char [{}] cannot be used for escaping", escape); } // lastly validate that escape chars (if present) are followed by special chars for (int i = 0; i < pattern.length(); i++) { @@ -324,11 +338,6 @@ public Order visitOrderBy(OrderByContext ctx) { ctx.DESC() != null ? Order.OrderDirection.DESC : Order.OrderDirection.ASC); } - @Override - public Object visitCast(CastContext ctx) { - return new Cast(source(ctx), expression(ctx.expression()), typedParsing(ctx.dataType(), DataType.class)); - } - @Override public DataType visitPrimitiveDataType(PrimitiveDataTypeContext ctx) { String type = visitIdentifier(ctx.identifier()).toLowerCase(Locale.ROOT); @@ -367,20 +376,32 @@ public DataType visitPrimitiveDataType(PrimitiveDataTypeContext ctx) { } } + // + // Functions template + // @Override - public Object visitFunctionCall(FunctionCallContext ctx) { - String name = visitIdentifier(ctx.identifier()); - boolean isDistinct = ctx.setQuantifier() != null && ctx.setQuantifier().DISTINCT() != null; - UnresolvedFunction.ResolutionType resolutionType = - isDistinct ? UnresolvedFunction.ResolutionType.DISTINCT : UnresolvedFunction.ResolutionType.STANDARD; - return new UnresolvedFunction(source(ctx), name, resolutionType, expressions(ctx.expression())); + public Cast visitCastExpression(CastExpressionContext ctx) { + CastTemplateContext ctc = ctx.castTemplate(); + return new Cast(source(ctc), expression(ctc.expression()), typedParsing(ctc.dataType(), DataType.class)); } @Override - public Object visitExtract(ExtractContext ctx) { - String fieldString = visitIdentifier(ctx.field); - return new UnresolvedFunction(source(ctx), fieldString, - UnresolvedFunction.ResolutionType.EXTRACT, singletonList(expression(ctx.valueExpression()))); + public Function visitExtractExpression(ExtractExpressionContext ctx) { + ExtractTemplateContext template = ctx.extractTemplate(); + String fieldString = visitIdentifier(template.field); + return new UnresolvedFunction(source(template), fieldString, + UnresolvedFunction.ResolutionType.EXTRACT, singletonList(expression(template.valueExpression()))); + } + + @Override + public Function visitFunctionExpression(FunctionExpressionContext ctx) { + FunctionTemplateContext template = ctx.functionTemplate(); + + String name = visitIdentifier(template.identifier()); + boolean isDistinct = template.setQuantifier() != null && template.setQuantifier().DISTINCT() != null; + UnresolvedFunction.ResolutionType resolutionType = + isDistinct ? UnresolvedFunction.ResolutionType.DISTINCT : UnresolvedFunction.ResolutionType.STANDARD; + return new UnresolvedFunction(source(ctx), name, resolutionType, expressions(template.expression())); } @Override @@ -445,12 +466,12 @@ public Expression visitStringLiteral(StringLiteralContext ctx) { } @Override - public Object visitDecimalLiteral(DecimalLiteralContext ctx) { + public Literal visitDecimalLiteral(DecimalLiteralContext ctx) { return new Literal(source(ctx), new BigDecimal(ctx.getText()).doubleValue(), DataType.DOUBLE); } @Override - public Object visitIntegerLiteral(IntegerLiteralContext ctx) { + public Literal visitIntegerLiteral(IntegerLiteralContext ctx) { BigDecimal bigD = new BigDecimal(ctx.getText()); long value = bigD.longValueExact(); @@ -463,7 +484,7 @@ public Object visitIntegerLiteral(IntegerLiteralContext ctx) { } @Override - public Object visitParamLiteral(ParamLiteralContext ctx) { + public Literal visitParamLiteral(ParamLiteralContext ctx) { SqlTypedParamValue param = param(ctx.PARAM()); Location loc = source(ctx); if (param.value == null) { @@ -522,4 +543,100 @@ private SqlTypedParamValue param(TerminalNode node) { return params.get(token); } -} + + @Override + public Literal visitDateEscapedLiteral(DateEscapedLiteralContext ctx) { + String string = string(ctx.string()); + Location loc = source(ctx); + // parse yyyy-MM-dd + DateTime dt = null; + try { + dt = ISODateTimeFormat.date().parseDateTime(string); + } catch(IllegalArgumentException ex) { + throw new ParsingException(loc, "Invalid date received; {}", ex.getMessage()); + } + return new Literal(loc, dt, DataType.DATE); + } + + @Override + public Literal visitTimeEscapedLiteral(TimeEscapedLiteralContext ctx) { + String string = string(ctx.string()); + Location loc = source(ctx); + + // parse HH:mm:ss + DateTime dt = null; + try { + dt = ISODateTimeFormat.hourMinuteSecond().parseDateTime(string); + } catch (IllegalArgumentException ex) { + throw new ParsingException(loc, "Invalid time received; {}", ex.getMessage()); + } + + throw new SqlIllegalArgumentException("Time (only) literals are not supported; a date component is required as well"); + } + + @Override + public Literal visitTimestampEscapedLiteral(TimestampEscapedLiteralContext ctx) { + String string = string(ctx.string()); + + Location loc = source(ctx); + // parse yyyy-mm-dd hh:mm:ss(.f...) + DateTime dt = null; + try { + DateTimeFormatter formatter = new DateTimeFormatterBuilder() + .append(ISODateTimeFormat.date()) + .appendLiteral(" ") + .append(ISODateTimeFormat.hourMinuteSecondFraction()) + .toFormatter(); + dt = formatter.parseDateTime(string); + } catch (IllegalArgumentException ex) { + throw new ParsingException(loc, "Invalid timestamp received; {}", ex.getMessage()); + } + return new Literal(loc, dt, DataType.DATE); + } + + @Override + public Literal visitGuidEscapedLiteral(GuidEscapedLiteralContext ctx) { + String string = string(ctx.string()); + + Location loc = source(ctx.string()); + // basic validation + String lowerCase = string.toLowerCase(Locale.ROOT); + // needs to be format nnnnnnnn-nnnn-nnnn-nnnn-nnnnnnnnnnnn + // since the length is fixed, the validation happens on absolute values + // not pretty but it's fast and doesn't create any extra objects + + String errorPrefix = "Invalid GUID, "; + + if (lowerCase.length() != 36) { + throw new ParsingException(loc, "{}too {}", errorPrefix, lowerCase.length() > 36 ? "long" : "short"); + } + + int[] separatorPos = { 8, 13, 18, 23 }; + for (int pos : separatorPos) { + if (lowerCase.charAt(pos) != '-') { + throw new ParsingException(loc, "{}expected group separator at offset [{}], found [{}]", + errorPrefix, pos, string.charAt(pos)); + } + } + + String HEXA = "0123456789abcdef"; + + for (int i = 0; i < lowerCase.length(); i++) { + // skip separators + boolean inspect = true; + for (int pos : separatorPos) { + if (i == pos) { + inspect = false; + break; + } else if (pos > i) { + break; + } + } + if (inspect && HEXA.indexOf(lowerCase.charAt(i)) < 0) { + throw new ParsingException(loc, "{}expected hexadecimal at offset[{}], found [{}]", errorPrefix, i, string.charAt(i)); + } + } + + return new Literal(source(ctx), string, DataType.KEYWORD); + } +} \ No newline at end of file diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/LogicalPlanBuilder.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/LogicalPlanBuilder.java index 3435994a0fc42..58d858c42415a 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/LogicalPlanBuilder.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/LogicalPlanBuilder.java @@ -19,6 +19,7 @@ import org.elasticsearch.xpack.sql.parser.SqlBaseParser.JoinCriteriaContext; import org.elasticsearch.xpack.sql.parser.SqlBaseParser.JoinRelationContext; import org.elasticsearch.xpack.sql.parser.SqlBaseParser.JoinTypeContext; +import org.elasticsearch.xpack.sql.parser.SqlBaseParser.LimitClauseContext; import org.elasticsearch.xpack.sql.parser.SqlBaseParser.NamedQueryContext; import org.elasticsearch.xpack.sql.parser.SqlBaseParser.QueryContext; import org.elasticsearch.xpack.sql.parser.SqlBaseParser.QueryNoWithContext; @@ -89,9 +90,13 @@ public LogicalPlan visitQueryNoWith(QueryNoWithContext ctx) { plan = new OrderBy(source(ctx.ORDER()), plan, visitList(ctx.orderBy(), Order.class)); } - if (ctx.limit != null && ctx.INTEGER_VALUE() != null) { - plan = new Limit(source(ctx.limit), new Literal(source(ctx), - Integer.parseInt(ctx.limit.getText()), DataType.INTEGER), plan); + LimitClauseContext limitClause = ctx.limitClause(); + if (limitClause != null) { + Token limit = limitClause.limit; + if (limit != null && limitClause.INTEGER_VALUE() != null) { + plan = new Limit(source(limitClause), new Literal(source(limitClause), + Integer.parseInt(limit.getText()), DataType.INTEGER), plan); + } } return plan; diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseBaseListener.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseBaseListener.java index 4e80e8db9bb52..b353bcf6521f9 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseBaseListener.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseBaseListener.java @@ -1,8 +1,3 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ // ANTLR GENERATED CODE: DO NOT EDIT package org.elasticsearch.xpack.sql.parser; @@ -208,6 +203,18 @@ class SqlBaseBaseListener implements SqlBaseListener { *

    The default implementation does nothing.

    */ @Override public void exitQueryNoWith(SqlBaseParser.QueryNoWithContext ctx) { } + /** + * {@inheritDoc} + * + *

    The default implementation does nothing.

    + */ + @Override public void enterLimitClause(SqlBaseParser.LimitClauseContext ctx) { } + /** + * {@inheritDoc} + * + *

    The default implementation does nothing.

    + */ + @Override public void exitLimitClause(SqlBaseParser.LimitClauseContext ctx) { } /** * {@inheritDoc} * @@ -556,6 +563,18 @@ class SqlBaseBaseListener implements SqlBaseListener { *

    The default implementation does nothing.

    */ @Override public void exitPattern(SqlBaseParser.PatternContext ctx) { } + /** + * {@inheritDoc} + * + *

    The default implementation does nothing.

    + */ + @Override public void enterPatternEscape(SqlBaseParser.PatternEscapeContext ctx) { } + /** + * {@inheritDoc} + * + *

    The default implementation does nothing.

    + */ + @Override public void exitPatternEscape(SqlBaseParser.PatternEscapeContext ctx) { } /** * {@inheritDoc} * @@ -657,13 +676,13 @@ class SqlBaseBaseListener implements SqlBaseListener { * *

    The default implementation does nothing.

    */ - @Override public void enterFunctionCall(SqlBaseParser.FunctionCallContext ctx) { } + @Override public void enterFunction(SqlBaseParser.FunctionContext ctx) { } /** * {@inheritDoc} * *

    The default implementation does nothing.

    */ - @Override public void exitFunctionCall(SqlBaseParser.FunctionCallContext ctx) { } + @Override public void exitFunction(SqlBaseParser.FunctionContext ctx) { } /** * {@inheritDoc} * @@ -712,6 +731,78 @@ class SqlBaseBaseListener implements SqlBaseListener { *

    The default implementation does nothing.

    */ @Override public void exitParenthesizedExpression(SqlBaseParser.ParenthesizedExpressionContext ctx) { } + /** + * {@inheritDoc} + * + *

    The default implementation does nothing.

    + */ + @Override public void enterCastExpression(SqlBaseParser.CastExpressionContext ctx) { } + /** + * {@inheritDoc} + * + *

    The default implementation does nothing.

    + */ + @Override public void exitCastExpression(SqlBaseParser.CastExpressionContext ctx) { } + /** + * {@inheritDoc} + * + *

    The default implementation does nothing.

    + */ + @Override public void enterCastTemplate(SqlBaseParser.CastTemplateContext ctx) { } + /** + * {@inheritDoc} + * + *

    The default implementation does nothing.

    + */ + @Override public void exitCastTemplate(SqlBaseParser.CastTemplateContext ctx) { } + /** + * {@inheritDoc} + * + *

    The default implementation does nothing.

    + */ + @Override public void enterExtractExpression(SqlBaseParser.ExtractExpressionContext ctx) { } + /** + * {@inheritDoc} + * + *

    The default implementation does nothing.

    + */ + @Override public void exitExtractExpression(SqlBaseParser.ExtractExpressionContext ctx) { } + /** + * {@inheritDoc} + * + *

    The default implementation does nothing.

    + */ + @Override public void enterExtractTemplate(SqlBaseParser.ExtractTemplateContext ctx) { } + /** + * {@inheritDoc} + * + *

    The default implementation does nothing.

    + */ + @Override public void exitExtractTemplate(SqlBaseParser.ExtractTemplateContext ctx) { } + /** + * {@inheritDoc} + * + *

    The default implementation does nothing.

    + */ + @Override public void enterFunctionExpression(SqlBaseParser.FunctionExpressionContext ctx) { } + /** + * {@inheritDoc} + * + *

    The default implementation does nothing.

    + */ + @Override public void exitFunctionExpression(SqlBaseParser.FunctionExpressionContext ctx) { } + /** + * {@inheritDoc} + * + *

    The default implementation does nothing.

    + */ + @Override public void enterFunctionTemplate(SqlBaseParser.FunctionTemplateContext ctx) { } + /** + * {@inheritDoc} + * + *

    The default implementation does nothing.

    + */ + @Override public void exitFunctionTemplate(SqlBaseParser.FunctionTemplateContext ctx) { } /** * {@inheritDoc} * @@ -772,6 +863,54 @@ class SqlBaseBaseListener implements SqlBaseListener { *

    The default implementation does nothing.

    */ @Override public void exitParamLiteral(SqlBaseParser.ParamLiteralContext ctx) { } + /** + * {@inheritDoc} + * + *

    The default implementation does nothing.

    + */ + @Override public void enterDateEscapedLiteral(SqlBaseParser.DateEscapedLiteralContext ctx) { } + /** + * {@inheritDoc} + * + *

    The default implementation does nothing.

    + */ + @Override public void exitDateEscapedLiteral(SqlBaseParser.DateEscapedLiteralContext ctx) { } + /** + * {@inheritDoc} + * + *

    The default implementation does nothing.

    + */ + @Override public void enterTimeEscapedLiteral(SqlBaseParser.TimeEscapedLiteralContext ctx) { } + /** + * {@inheritDoc} + * + *

    The default implementation does nothing.

    + */ + @Override public void exitTimeEscapedLiteral(SqlBaseParser.TimeEscapedLiteralContext ctx) { } + /** + * {@inheritDoc} + * + *

    The default implementation does nothing.

    + */ + @Override public void enterTimestampEscapedLiteral(SqlBaseParser.TimestampEscapedLiteralContext ctx) { } + /** + * {@inheritDoc} + * + *

    The default implementation does nothing.

    + */ + @Override public void exitTimestampEscapedLiteral(SqlBaseParser.TimestampEscapedLiteralContext ctx) { } + /** + * {@inheritDoc} + * + *

    The default implementation does nothing.

    + */ + @Override public void enterGuidEscapedLiteral(SqlBaseParser.GuidEscapedLiteralContext ctx) { } + /** + * {@inheritDoc} + * + *

    The default implementation does nothing.

    + */ + @Override public void exitGuidEscapedLiteral(SqlBaseParser.GuidEscapedLiteralContext ctx) { } /** * {@inheritDoc} * diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseBaseVisitor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseBaseVisitor.java index 1adb0a423c7f2..d40ae6daa6e34 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseBaseVisitor.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseBaseVisitor.java @@ -1,8 +1,3 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ // ANTLR GENERATED CODE: DO NOT EDIT package org.elasticsearch.xpack.sql.parser; import org.antlr.v4.runtime.tree.AbstractParseTreeVisitor; @@ -128,6 +123,13 @@ class SqlBaseBaseVisitor extends AbstractParseTreeVisitor implements SqlBa * {@link #visitChildren} on {@code ctx}.

    */ @Override public T visitQueryNoWith(SqlBaseParser.QueryNoWithContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitLimitClause(SqlBaseParser.LimitClauseContext ctx) { return visitChildren(ctx); } /** * {@inheritDoc} * @@ -331,6 +333,13 @@ class SqlBaseBaseVisitor extends AbstractParseTreeVisitor implements SqlBa * {@link #visitChildren} on {@code ctx}.

    */ @Override public T visitPattern(SqlBaseParser.PatternContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitPatternEscape(SqlBaseParser.PatternEscapeContext ctx) { return visitChildren(ctx); } /** * {@inheritDoc} * @@ -393,7 +402,7 @@ class SqlBaseBaseVisitor extends AbstractParseTreeVisitor implements SqlBa *

    The default implementation returns the result of calling * {@link #visitChildren} on {@code ctx}.

    */ - @Override public T visitFunctionCall(SqlBaseParser.FunctionCallContext ctx) { return visitChildren(ctx); } + @Override public T visitFunction(SqlBaseParser.FunctionContext ctx) { return visitChildren(ctx); } /** * {@inheritDoc} * @@ -422,6 +431,48 @@ class SqlBaseBaseVisitor extends AbstractParseTreeVisitor implements SqlBa * {@link #visitChildren} on {@code ctx}.

    */ @Override public T visitParenthesizedExpression(SqlBaseParser.ParenthesizedExpressionContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitCastExpression(SqlBaseParser.CastExpressionContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitCastTemplate(SqlBaseParser.CastTemplateContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitExtractExpression(SqlBaseParser.ExtractExpressionContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitExtractTemplate(SqlBaseParser.ExtractTemplateContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitFunctionExpression(SqlBaseParser.FunctionExpressionContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitFunctionTemplate(SqlBaseParser.FunctionTemplateContext ctx) { return visitChildren(ctx); } /** * {@inheritDoc} * @@ -457,6 +508,34 @@ class SqlBaseBaseVisitor extends AbstractParseTreeVisitor implements SqlBa * {@link #visitChildren} on {@code ctx}.

    */ @Override public T visitParamLiteral(SqlBaseParser.ParamLiteralContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitDateEscapedLiteral(SqlBaseParser.DateEscapedLiteralContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitTimeEscapedLiteral(SqlBaseParser.TimeEscapedLiteralContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitTimestampEscapedLiteral(SqlBaseParser.TimestampEscapedLiteralContext ctx) { return visitChildren(ctx); } + /** + * {@inheritDoc} + * + *

    The default implementation returns the result of calling + * {@link #visitChildren} on {@code ctx}.

    + */ + @Override public T visitGuidEscapedLiteral(SqlBaseParser.GuidEscapedLiteralContext ctx) { return visitChildren(ctx); } /** * {@inheritDoc} * diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseLexer.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseLexer.java index c54c5e3810c8f..588f3ef028d6f 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseLexer.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseLexer.java @@ -1,15 +1,13 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ // ANTLR GENERATED CODE: DO NOT EDIT package org.elasticsearch.xpack.sql.parser; import org.antlr.v4.runtime.Lexer; import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.TokenStream; import org.antlr.v4.runtime.*; import org.antlr.v4.runtime.atn.*; import org.antlr.v4.runtime.dfa.DFA; +import org.antlr.v4.runtime.misc.*; @SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"}) class SqlBaseLexer extends Lexer { @@ -28,11 +26,13 @@ class SqlBaseLexer extends Lexer { NOT=45, NULL=46, ON=47, OPTIMIZED=48, OR=49, ORDER=50, OUTER=51, PARSED=52, PHYSICAL=53, PLAN=54, RIGHT=55, RLIKE=56, QUERY=57, SCHEMAS=58, SELECT=59, SHOW=60, SYS=61, TABLE=62, TABLES=63, TEXT=64, TRUE=65, TYPE=66, TYPES=67, - USING=68, VERIFY=69, WHERE=70, WITH=71, EQ=72, NEQ=73, LT=74, LTE=75, - GT=76, GTE=77, PLUS=78, MINUS=79, ASTERISK=80, SLASH=81, PERCENT=82, CONCAT=83, - DOT=84, PARAM=85, STRING=86, INTEGER_VALUE=87, DECIMAL_VALUE=88, IDENTIFIER=89, - DIGIT_IDENTIFIER=90, TABLE_IDENTIFIER=91, QUOTED_IDENTIFIER=92, BACKQUOTED_IDENTIFIER=93, - SIMPLE_COMMENT=94, BRACKETED_COMMENT=95, WS=96, UNRECOGNIZED=97; + USING=68, VERIFY=69, WHERE=70, WITH=71, ESCAPE_ESC=72, FUNCTION_ESC=73, + LIMIT_ESC=74, DATE_ESC=75, TIME_ESC=76, TIMESTAMP_ESC=77, GUID_ESC=78, + ESC_END=79, EQ=80, NEQ=81, LT=82, LTE=83, GT=84, GTE=85, PLUS=86, MINUS=87, + ASTERISK=88, SLASH=89, PERCENT=90, CONCAT=91, DOT=92, PARAM=93, STRING=94, + INTEGER_VALUE=95, DECIMAL_VALUE=96, IDENTIFIER=97, DIGIT_IDENTIFIER=98, + TABLE_IDENTIFIER=99, QUOTED_IDENTIFIER=100, BACKQUOTED_IDENTIFIER=101, + SIMPLE_COMMENT=102, BRACKETED_COMMENT=103, WS=104, UNRECOGNIZED=105; public static String[] modeNames = { "DEFAULT_MODE" }; @@ -46,12 +46,13 @@ class SqlBaseLexer extends Lexer { "LIMIT", "MAPPED", "MATCH", "NATURAL", "NOT", "NULL", "ON", "OPTIMIZED", "OR", "ORDER", "OUTER", "PARSED", "PHYSICAL", "PLAN", "RIGHT", "RLIKE", "QUERY", "SCHEMAS", "SELECT", "SHOW", "SYS", "TABLE", "TABLES", "TEXT", - "TRUE", "TYPE", "TYPES", "USING", "VERIFY", "WHERE", "WITH", "EQ", "NEQ", - "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", "SLASH", "PERCENT", - "CONCAT", "DOT", "PARAM", "STRING", "INTEGER_VALUE", "DECIMAL_VALUE", - "IDENTIFIER", "DIGIT_IDENTIFIER", "TABLE_IDENTIFIER", "QUOTED_IDENTIFIER", - "BACKQUOTED_IDENTIFIER", "EXPONENT", "DIGIT", "LETTER", "SIMPLE_COMMENT", - "BRACKETED_COMMENT", "WS", "UNRECOGNIZED" + "TRUE", "TYPE", "TYPES", "USING", "VERIFY", "WHERE", "WITH", "ESCAPE_ESC", + "FUNCTION_ESC", "LIMIT_ESC", "DATE_ESC", "TIME_ESC", "TIMESTAMP_ESC", + "GUID_ESC", "ESC_END", "EQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", + "MINUS", "ASTERISK", "SLASH", "PERCENT", "CONCAT", "DOT", "PARAM", "STRING", + "INTEGER_VALUE", "DECIMAL_VALUE", "IDENTIFIER", "DIGIT_IDENTIFIER", "TABLE_IDENTIFIER", + "QUOTED_IDENTIFIER", "BACKQUOTED_IDENTIFIER", "EXPONENT", "DIGIT", "LETTER", + "SIMPLE_COMMENT", "BRACKETED_COMMENT", "WS", "UNRECOGNIZED" }; private static final String[] _LITERAL_NAMES = { @@ -65,8 +66,9 @@ class SqlBaseLexer extends Lexer { "'OR'", "'ORDER'", "'OUTER'", "'PARSED'", "'PHYSICAL'", "'PLAN'", "'RIGHT'", "'RLIKE'", "'QUERY'", "'SCHEMAS'", "'SELECT'", "'SHOW'", "'SYS'", "'TABLE'", "'TABLES'", "'TEXT'", "'TRUE'", "'TYPE'", "'TYPES'", "'USING'", "'VERIFY'", - "'WHERE'", "'WITH'", "'='", null, "'<'", "'<='", "'>'", "'>='", "'+'", - "'-'", "'*'", "'/'", "'%'", "'||'", "'.'", "'?'" + "'WHERE'", "'WITH'", "'{ESCAPE'", "'{FN'", "'{LIMIT'", "'{D'", "'{T'", + "'{TS'", "'{GUID'", "'}'", "'='", null, "'<'", "'<='", "'>'", "'>='", + "'+'", "'-'", "'*'", "'/'", "'%'", "'||'", "'.'", "'?'" }; private static final String[] _SYMBOLIC_NAMES = { null, null, null, null, null, "ALL", "ANALYZE", "ANALYZED", "AND", "ANY", @@ -77,12 +79,13 @@ class SqlBaseLexer extends Lexer { "LIMIT", "MAPPED", "MATCH", "NATURAL", "NOT", "NULL", "ON", "OPTIMIZED", "OR", "ORDER", "OUTER", "PARSED", "PHYSICAL", "PLAN", "RIGHT", "RLIKE", "QUERY", "SCHEMAS", "SELECT", "SHOW", "SYS", "TABLE", "TABLES", "TEXT", - "TRUE", "TYPE", "TYPES", "USING", "VERIFY", "WHERE", "WITH", "EQ", "NEQ", - "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", "SLASH", "PERCENT", - "CONCAT", "DOT", "PARAM", "STRING", "INTEGER_VALUE", "DECIMAL_VALUE", - "IDENTIFIER", "DIGIT_IDENTIFIER", "TABLE_IDENTIFIER", "QUOTED_IDENTIFIER", - "BACKQUOTED_IDENTIFIER", "SIMPLE_COMMENT", "BRACKETED_COMMENT", "WS", - "UNRECOGNIZED" + "TRUE", "TYPE", "TYPES", "USING", "VERIFY", "WHERE", "WITH", "ESCAPE_ESC", + "FUNCTION_ESC", "LIMIT_ESC", "DATE_ESC", "TIME_ESC", "TIMESTAMP_ESC", + "GUID_ESC", "ESC_END", "EQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", + "MINUS", "ASTERISK", "SLASH", "PERCENT", "CONCAT", "DOT", "PARAM", "STRING", + "INTEGER_VALUE", "DECIMAL_VALUE", "IDENTIFIER", "DIGIT_IDENTIFIER", "TABLE_IDENTIFIER", + "QUOTED_IDENTIFIER", "BACKQUOTED_IDENTIFIER", "SIMPLE_COMMENT", "BRACKETED_COMMENT", + "WS", "UNRECOGNIZED" }; public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES); @@ -139,7 +142,7 @@ public SqlBaseLexer(CharStream input) { public ATN getATN() { return _ATN; } public static final String _serializedATN = - "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\2c\u033b\b\1\4\2\t"+ + "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\2k\u0370\b\1\4\2\t"+ "\2\4\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4\13"+ "\t\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22\t\22"+ "\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31\t\31"+ @@ -150,276 +153,293 @@ public SqlBaseLexer(CharStream input) { "\4>\t>\4?\t?\4@\t@\4A\tA\4B\tB\4C\tC\4D\tD\4E\tE\4F\tF\4G\tG\4H\tH\4I"+ "\tI\4J\tJ\4K\tK\4L\tL\4M\tM\4N\tN\4O\tO\4P\tP\4Q\tQ\4R\tR\4S\tS\4T\tT"+ "\4U\tU\4V\tV\4W\tW\4X\tX\4Y\tY\4Z\tZ\4[\t[\4\\\t\\\4]\t]\4^\t^\4_\t_\4"+ - "`\t`\4a\ta\4b\tb\4c\tc\4d\td\4e\te\3\2\3\2\3\3\3\3\3\4\3\4\3\5\3\5\3\6"+ - "\3\6\3\6\3\6\3\7\3\7\3\7\3\7\3\7\3\7\3\7\3\7\3\b\3\b\3\b\3\b\3\b\3\b\3"+ - "\b\3\b\3\b\3\t\3\t\3\t\3\t\3\n\3\n\3\n\3\n\3\13\3\13\3\13\3\f\3\f\3\f"+ - "\3\f\3\r\3\r\3\r\3\r\3\r\3\r\3\r\3\r\3\16\3\16\3\16\3\17\3\17\3\17\3\17"+ - "\3\17\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\20\3\21\3\21\3\21\3\21\3\21"+ - "\3\21\3\21\3\21\3\21\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\22\3\23\3\23"+ - "\3\23\3\23\3\23\3\23\3\24\3\24\3\24\3\24\3\24\3\25\3\25\3\25\3\25\3\25"+ - "\3\25\3\25\3\25\3\25\3\26\3\26\3\26\3\26\3\26\3\26\3\26\3\26\3\26\3\27"+ - "\3\27\3\27\3\27\3\27\3\27\3\27\3\30\3\30\3\30\3\30\3\30\3\30\3\30\3\30"+ - "\3\30\3\30\3\30\3\31\3\31\3\31\3\31\3\31\3\31\3\31\3\32\3\32\3\32\3\32"+ - "\3\32\3\32\3\32\3\32\3\33\3\33\3\33\3\33\3\33\3\33\3\33\3\33\3\34\3\34"+ - "\3\34\3\34\3\34\3\34\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3\36\3\36\3\36"+ - "\3\36\3\36\3\37\3\37\3\37\3\37\3\37\3 \3 \3 \3 \3 \3 \3 \3 \3 \3 \3!\3"+ - "!\3!\3!\3!\3!\3!\3!\3!\3\"\3\"\3\"\3\"\3\"\3\"\3#\3#\3#\3#\3#\3#\3#\3"+ - "$\3$\3$\3%\3%\3%\3%\3%\3%\3&\3&\3&\3\'\3\'\3\'\3\'\3\'\3(\3(\3(\3(\3("+ - "\3)\3)\3)\3)\3)\3*\3*\3*\3*\3*\3*\3+\3+\3+\3+\3+\3+\3+\3,\3,\3,\3,\3,"+ - "\3,\3-\3-\3-\3-\3-\3-\3-\3-\3.\3.\3.\3.\3/\3/\3/\3/\3/\3\60\3\60\3\60"+ - "\3\61\3\61\3\61\3\61\3\61\3\61\3\61\3\61\3\61\3\61\3\62\3\62\3\62\3\63"+ - "\3\63\3\63\3\63\3\63\3\63\3\64\3\64\3\64\3\64\3\64\3\64\3\65\3\65\3\65"+ - "\3\65\3\65\3\65\3\65\3\66\3\66\3\66\3\66\3\66\3\66\3\66\3\66\3\66\3\67"+ - "\3\67\3\67\3\67\3\67\38\38\38\38\38\38\39\39\39\39\39\39\3:\3:\3:\3:\3"+ - ":\3:\3;\3;\3;\3;\3;\3;\3;\3;\3<\3<\3<\3<\3<\3<\3<\3=\3=\3=\3=\3=\3>\3"+ - ">\3>\3>\3?\3?\3?\3?\3?\3?\3@\3@\3@\3@\3@\3@\3@\3A\3A\3A\3A\3A\3B\3B\3"+ - "B\3B\3B\3C\3C\3C\3C\3C\3D\3D\3D\3D\3D\3D\3E\3E\3E\3E\3E\3E\3F\3F\3F\3"+ - "F\3F\3F\3F\3G\3G\3G\3G\3G\3G\3H\3H\3H\3H\3H\3I\3I\3J\3J\3J\3J\3J\3J\3"+ - "J\5J\u027b\nJ\3K\3K\3L\3L\3L\3M\3M\3N\3N\3N\3O\3O\3P\3P\3Q\3Q\3R\3R\3"+ - "S\3S\3T\3T\3T\3U\3U\3V\3V\3W\3W\3W\3W\7W\u029c\nW\fW\16W\u029f\13W\3W"+ - "\3W\3X\6X\u02a4\nX\rX\16X\u02a5\3Y\6Y\u02a9\nY\rY\16Y\u02aa\3Y\3Y\7Y\u02af"+ - "\nY\fY\16Y\u02b2\13Y\3Y\3Y\6Y\u02b6\nY\rY\16Y\u02b7\3Y\6Y\u02bb\nY\rY"+ - "\16Y\u02bc\3Y\3Y\7Y\u02c1\nY\fY\16Y\u02c4\13Y\5Y\u02c6\nY\3Y\3Y\3Y\3Y"+ - "\6Y\u02cc\nY\rY\16Y\u02cd\3Y\3Y\5Y\u02d2\nY\3Z\3Z\5Z\u02d6\nZ\3Z\3Z\3"+ - "Z\7Z\u02db\nZ\fZ\16Z\u02de\13Z\3[\3[\3[\3[\6[\u02e4\n[\r[\16[\u02e5\3"+ - "\\\3\\\3\\\3\\\6\\\u02ec\n\\\r\\\16\\\u02ed\3]\3]\3]\3]\7]\u02f4\n]\f"+ - "]\16]\u02f7\13]\3]\3]\3^\3^\3^\3^\7^\u02ff\n^\f^\16^\u0302\13^\3^\3^\3"+ - "_\3_\5_\u0308\n_\3_\6_\u030b\n_\r_\16_\u030c\3`\3`\3a\3a\3b\3b\3b\3b\7"+ - "b\u0317\nb\fb\16b\u031a\13b\3b\5b\u031d\nb\3b\5b\u0320\nb\3b\3b\3c\3c"+ - "\3c\3c\3c\7c\u0329\nc\fc\16c\u032c\13c\3c\3c\3c\3c\3c\3d\6d\u0334\nd\r"+ - "d\16d\u0335\3d\3d\3e\3e\3\u032a\2f\3\3\5\4\7\5\t\6\13\7\r\b\17\t\21\n"+ - "\23\13\25\f\27\r\31\16\33\17\35\20\37\21!\22#\23%\24\'\25)\26+\27-\30"+ - "/\31\61\32\63\33\65\34\67\359\36;\37= ?!A\"C#E$G%I&K\'M(O)Q*S+U,W-Y.["+ - "/]\60_\61a\62c\63e\64g\65i\66k\67m8o9q:s;u{?}@\177A\u0081B\u0083"+ - "C\u0085D\u0087E\u0089F\u008bG\u008dH\u008fI\u0091J\u0093K\u0095L\u0097"+ - "M\u0099N\u009bO\u009dP\u009fQ\u00a1R\u00a3S\u00a5T\u00a7U\u00a9V\u00ab"+ - "W\u00adX\u00afY\u00b1Z\u00b3[\u00b5\\\u00b7]\u00b9^\u00bb_\u00bd\2\u00bf"+ - "\2\u00c1\2\u00c3`\u00c5a\u00c7b\u00c9c\3\2\f\3\2))\4\2BBaa\5\2<\3>\3>\3>\3?\3?\3?\3?"+ + "\3?\3?\3@\3@\3@\3@\3@\3@\3@\3A\3A\3A\3A\3A\3B\3B\3B\3B\3B\3C\3C\3C\3C"+ + "\3C\3D\3D\3D\3D\3D\3D\3E\3E\3E\3E\3E\3E\3F\3F\3F\3F\3F\3F\3F\3G\3G\3G"+ + "\3G\3G\3G\3H\3H\3H\3H\3H\3I\3I\3I\3I\3I\3I\3I\3I\3J\3J\3J\3J\3K\3K\3K"+ + "\3K\3K\3K\3K\3L\3L\3L\3M\3M\3M\3N\3N\3N\3N\3O\3O\3O\3O\3O\3O\3P\3P\3Q"+ + "\3Q\3R\3R\3R\3R\3R\3R\3R\5R\u02b0\nR\3S\3S\3T\3T\3T\3U\3U\3V\3V\3V\3W"+ + "\3W\3X\3X\3Y\3Y\3Z\3Z\3[\3[\3\\\3\\\3\\\3]\3]\3^\3^\3_\3_\3_\3_\7_\u02d1"+ + "\n_\f_\16_\u02d4\13_\3_\3_\3`\6`\u02d9\n`\r`\16`\u02da\3a\6a\u02de\na"+ + "\ra\16a\u02df\3a\3a\7a\u02e4\na\fa\16a\u02e7\13a\3a\3a\6a\u02eb\na\ra"+ + "\16a\u02ec\3a\6a\u02f0\na\ra\16a\u02f1\3a\3a\7a\u02f6\na\fa\16a\u02f9"+ + "\13a\5a\u02fb\na\3a\3a\3a\3a\6a\u0301\na\ra\16a\u0302\3a\3a\5a\u0307\n"+ + "a\3b\3b\5b\u030b\nb\3b\3b\3b\7b\u0310\nb\fb\16b\u0313\13b\3c\3c\3c\3c"+ + "\6c\u0319\nc\rc\16c\u031a\3d\3d\3d\3d\6d\u0321\nd\rd\16d\u0322\3e\3e\3"+ + "e\3e\7e\u0329\ne\fe\16e\u032c\13e\3e\3e\3f\3f\3f\3f\7f\u0334\nf\ff\16"+ + "f\u0337\13f\3f\3f\3g\3g\5g\u033d\ng\3g\6g\u0340\ng\rg\16g\u0341\3h\3h"+ + "\3i\3i\3j\3j\3j\3j\7j\u034c\nj\fj\16j\u034f\13j\3j\5j\u0352\nj\3j\5j\u0355"+ + "\nj\3j\3j\3k\3k\3k\3k\3k\7k\u035e\nk\fk\16k\u0361\13k\3k\3k\3k\3k\3k\3"+ + "l\6l\u0369\nl\rl\16l\u036a\3l\3l\3m\3m\3\u035f\2n\3\3\5\4\7\5\t\6\13\7"+ + "\r\b\17\t\21\n\23\13\25\f\27\r\31\16\33\17\35\20\37\21!\22#\23%\24\'\25"+ + ")\26+\27-\30/\31\61\32\63\33\65\34\67\359\36;\37= ?!A\"C#E$G%I&K\'M(O"+ + ")Q*S+U,W-Y.[/]\60_\61a\62c\63e\64g\65i\66k\67m8o9q:s;u{?}@\177A\u0081"+ + "B\u0083C\u0085D\u0087E\u0089F\u008bG\u008dH\u008fI\u0091J\u0093K\u0095"+ + "L\u0097M\u0099N\u009bO\u009dP\u009fQ\u00a1R\u00a3S\u00a5T\u00a7U\u00a9"+ + "V\u00abW\u00adX\u00afY\u00b1Z\u00b3[\u00b5\\\u00b7]\u00b9^\u00bb_\u00bd"+ + "`\u00bfa\u00c1b\u00c3c\u00c5d\u00c7e\u00c9f\u00cbg\u00cd\2\u00cf\2\u00d1"+ + "\2\u00d3h\u00d5i\u00d7j\u00d9k\3\2\f\3\2))\4\2BBaa\5\2<\3\2\2\2\u017d\u017e"+ - "\7H\2\2\u017e\u017f\7W\2\2\u017f\u0180\7P\2\2\u0180\u0181\7E\2\2\u0181"+ - "\u0182\7V\2\2\u0182\u0183\7K\2\2\u0183\u0184\7Q\2\2\u0184\u0185\7P\2\2"+ - "\u0185\u0186\7U\2\2\u0186@\3\2\2\2\u0187\u0188\7I\2\2\u0188\u0189\7T\2"+ - "\2\u0189\u018a\7C\2\2\u018a\u018b\7R\2\2\u018b\u018c\7J\2\2\u018c\u018d"+ - "\7X\2\2\u018d\u018e\7K\2\2\u018e\u018f\7\\\2\2\u018fB\3\2\2\2\u0190\u0191"+ - "\7I\2\2\u0191\u0192\7T\2\2\u0192\u0193\7Q\2\2\u0193\u0194\7W\2\2\u0194"+ - "\u0195\7R\2\2\u0195D\3\2\2\2\u0196\u0197\7J\2\2\u0197\u0198\7C\2\2\u0198"+ - "\u0199\7X\2\2\u0199\u019a\7K\2\2\u019a\u019b\7P\2\2\u019b\u019c\7I\2\2"+ - "\u019cF\3\2\2\2\u019d\u019e\7K\2\2\u019e\u019f\7P\2\2\u019fH\3\2\2\2\u01a0"+ - "\u01a1\7K\2\2\u01a1\u01a2\7P\2\2\u01a2\u01a3\7P\2\2\u01a3\u01a4\7G\2\2"+ - "\u01a4\u01a5\7T\2\2\u01a5J\3\2\2\2\u01a6\u01a7\7K\2\2\u01a7\u01a8\7U\2"+ - "\2\u01a8L\3\2\2\2\u01a9\u01aa\7L\2\2\u01aa\u01ab\7Q\2\2\u01ab\u01ac\7"+ - "K\2\2\u01ac\u01ad\7P\2\2\u01adN\3\2\2\2\u01ae\u01af\7N\2\2\u01af\u01b0"+ - "\7G\2\2\u01b0\u01b1\7H\2\2\u01b1\u01b2\7V\2\2\u01b2P\3\2\2\2\u01b3\u01b4"+ - "\7N\2\2\u01b4\u01b5\7K\2\2\u01b5\u01b6\7M\2\2\u01b6\u01b7\7G\2\2\u01b7"+ - "R\3\2\2\2\u01b8\u01b9\7N\2\2\u01b9\u01ba\7K\2\2\u01ba\u01bb\7O\2\2\u01bb"+ - "\u01bc\7K\2\2\u01bc\u01bd\7V\2\2\u01bdT\3\2\2\2\u01be\u01bf\7O\2\2\u01bf"+ - "\u01c0\7C\2\2\u01c0\u01c1\7R\2\2\u01c1\u01c2\7R\2\2\u01c2\u01c3\7G\2\2"+ - "\u01c3\u01c4\7F\2\2\u01c4V\3\2\2\2\u01c5\u01c6\7O\2\2\u01c6\u01c7\7C\2"+ - "\2\u01c7\u01c8\7V\2\2\u01c8\u01c9\7E\2\2\u01c9\u01ca\7J\2\2\u01caX\3\2"+ - "\2\2\u01cb\u01cc\7P\2\2\u01cc\u01cd\7C\2\2\u01cd\u01ce\7V\2\2\u01ce\u01cf"+ - "\7W\2\2\u01cf\u01d0\7T\2\2\u01d0\u01d1\7C\2\2\u01d1\u01d2\7N\2\2\u01d2"+ - "Z\3\2\2\2\u01d3\u01d4\7P\2\2\u01d4\u01d5\7Q\2\2\u01d5\u01d6\7V\2\2\u01d6"+ - "\\\3\2\2\2\u01d7\u01d8\7P\2\2\u01d8\u01d9\7W\2\2\u01d9\u01da\7N\2\2\u01da"+ - "\u01db\7N\2\2\u01db^\3\2\2\2\u01dc\u01dd\7Q\2\2\u01dd\u01de\7P\2\2\u01de"+ - "`\3\2\2\2\u01df\u01e0\7Q\2\2\u01e0\u01e1\7R\2\2\u01e1\u01e2\7V\2\2\u01e2"+ - "\u01e3\7K\2\2\u01e3\u01e4\7O\2\2\u01e4\u01e5\7K\2\2\u01e5\u01e6\7\\\2"+ - "\2\u01e6\u01e7\7G\2\2\u01e7\u01e8\7F\2\2\u01e8b\3\2\2\2\u01e9\u01ea\7"+ - "Q\2\2\u01ea\u01eb\7T\2\2\u01ebd\3\2\2\2\u01ec\u01ed\7Q\2\2\u01ed\u01ee"+ - "\7T\2\2\u01ee\u01ef\7F\2\2\u01ef\u01f0\7G\2\2\u01f0\u01f1\7T\2\2\u01f1"+ - "f\3\2\2\2\u01f2\u01f3\7Q\2\2\u01f3\u01f4\7W\2\2\u01f4\u01f5\7V\2\2\u01f5"+ - "\u01f6\7G\2\2\u01f6\u01f7\7T\2\2\u01f7h\3\2\2\2\u01f8\u01f9\7R\2\2\u01f9"+ - "\u01fa\7C\2\2\u01fa\u01fb\7T\2\2\u01fb\u01fc\7U\2\2\u01fc\u01fd\7G\2\2"+ - "\u01fd\u01fe\7F\2\2\u01fej\3\2\2\2\u01ff\u0200\7R\2\2\u0200\u0201\7J\2"+ - "\2\u0201\u0202\7[\2\2\u0202\u0203\7U\2\2\u0203\u0204\7K\2\2\u0204\u0205"+ - "\7E\2\2\u0205\u0206\7C\2\2\u0206\u0207\7N\2\2\u0207l\3\2\2\2\u0208\u0209"+ - "\7R\2\2\u0209\u020a\7N\2\2\u020a\u020b\7C\2\2\u020b\u020c\7P\2\2\u020c"+ - "n\3\2\2\2\u020d\u020e\7T\2\2\u020e\u020f\7K\2\2\u020f\u0210\7I\2\2\u0210"+ - "\u0211\7J\2\2\u0211\u0212\7V\2\2\u0212p\3\2\2\2\u0213\u0214\7T\2\2\u0214"+ - "\u0215\7N\2\2\u0215\u0216\7K\2\2\u0216\u0217\7M\2\2\u0217\u0218\7G\2\2"+ - "\u0218r\3\2\2\2\u0219\u021a\7S\2\2\u021a\u021b\7W\2\2\u021b\u021c\7G\2"+ - "\2\u021c\u021d\7T\2\2\u021d\u021e\7[\2\2\u021et\3\2\2\2\u021f\u0220\7"+ - "U\2\2\u0220\u0221\7E\2\2\u0221\u0222\7J\2\2\u0222\u0223\7G\2\2\u0223\u0224"+ - "\7O\2\2\u0224\u0225\7C\2\2\u0225\u0226\7U\2\2\u0226v\3\2\2\2\u0227\u0228"+ - "\7U\2\2\u0228\u0229\7G\2\2\u0229\u022a\7N\2\2\u022a\u022b\7G\2\2\u022b"+ - "\u022c\7E\2\2\u022c\u022d\7V\2\2\u022dx\3\2\2\2\u022e\u022f\7U\2\2\u022f"+ - "\u0230\7J\2\2\u0230\u0231\7Q\2\2\u0231\u0232\7Y\2\2\u0232z\3\2\2\2\u0233"+ - "\u0234\7U\2\2\u0234\u0235\7[\2\2\u0235\u0236\7U\2\2\u0236|\3\2\2\2\u0237"+ - "\u0238\7V\2\2\u0238\u0239\7C\2\2\u0239\u023a\7D\2\2\u023a\u023b\7N\2\2"+ - "\u023b\u023c\7G\2\2\u023c~\3\2\2\2\u023d\u023e\7V\2\2\u023e\u023f\7C\2"+ - "\2\u023f\u0240\7D\2\2\u0240\u0241\7N\2\2\u0241\u0242\7G\2\2\u0242\u0243"+ - "\7U\2\2\u0243\u0080\3\2\2\2\u0244\u0245\7V\2\2\u0245\u0246\7G\2\2\u0246"+ - "\u0247\7Z\2\2\u0247\u0248\7V\2\2\u0248\u0082\3\2\2\2\u0249\u024a\7V\2"+ - "\2\u024a\u024b\7T\2\2\u024b\u024c\7W\2\2\u024c\u024d\7G\2\2\u024d\u0084"+ - "\3\2\2\2\u024e\u024f\7V\2\2\u024f\u0250\7[\2\2\u0250\u0251\7R\2\2\u0251"+ - "\u0252\7G\2\2\u0252\u0086\3\2\2\2\u0253\u0254\7V\2\2\u0254\u0255\7[\2"+ - "\2\u0255\u0256\7R\2\2\u0256\u0257\7G\2\2\u0257\u0258\7U\2\2\u0258\u0088"+ - "\3\2\2\2\u0259\u025a\7W\2\2\u025a\u025b\7U\2\2\u025b\u025c\7K\2\2\u025c"+ - "\u025d\7P\2\2\u025d\u025e\7I\2\2\u025e\u008a\3\2\2\2\u025f\u0260\7X\2"+ - "\2\u0260\u0261\7G\2\2\u0261\u0262\7T\2\2\u0262\u0263\7K\2\2\u0263\u0264"+ - "\7H\2\2\u0264\u0265\7[\2\2\u0265\u008c\3\2\2\2\u0266\u0267\7Y\2\2\u0267"+ - "\u0268\7J\2\2\u0268\u0269\7G\2\2\u0269\u026a\7T\2\2\u026a\u026b\7G\2\2"+ - "\u026b\u008e\3\2\2\2\u026c\u026d\7Y\2\2\u026d\u026e\7K\2\2\u026e\u026f"+ - "\7V\2\2\u026f\u0270\7J\2\2\u0270\u0090\3\2\2\2\u0271\u0272\7?\2\2\u0272"+ - "\u0092\3\2\2\2\u0273\u0274\7>\2\2\u0274\u027b\7@\2\2\u0275\u0276\7#\2"+ - "\2\u0276\u027b\7?\2\2\u0277\u0278\7>\2\2\u0278\u0279\7?\2\2\u0279\u027b"+ - "\7@\2\2\u027a\u0273\3\2\2\2\u027a\u0275\3\2\2\2\u027a\u0277\3\2\2\2\u027b"+ - "\u0094\3\2\2\2\u027c\u027d\7>\2\2\u027d\u0096\3\2\2\2\u027e\u027f\7>\2"+ - "\2\u027f\u0280\7?\2\2\u0280\u0098\3\2\2\2\u0281\u0282\7@\2\2\u0282\u009a"+ - "\3\2\2\2\u0283\u0284\7@\2\2\u0284\u0285\7?\2\2\u0285\u009c\3\2\2\2\u0286"+ - "\u0287\7-\2\2\u0287\u009e\3\2\2\2\u0288\u0289\7/\2\2\u0289\u00a0\3\2\2"+ - "\2\u028a\u028b\7,\2\2\u028b\u00a2\3\2\2\2\u028c\u028d\7\61\2\2\u028d\u00a4"+ - "\3\2\2\2\u028e\u028f\7\'\2\2\u028f\u00a6\3\2\2\2\u0290\u0291\7~\2\2\u0291"+ - "\u0292\7~\2\2\u0292\u00a8\3\2\2\2\u0293\u0294\7\60\2\2\u0294\u00aa\3\2"+ - "\2\2\u0295\u0296\7A\2\2\u0296\u00ac\3\2\2\2\u0297\u029d\7)\2\2\u0298\u029c"+ - "\n\2\2\2\u0299\u029a\7)\2\2\u029a\u029c\7)\2\2\u029b\u0298\3\2\2\2\u029b"+ - "\u0299\3\2\2\2\u029c\u029f\3\2\2\2\u029d\u029b\3\2\2\2\u029d\u029e\3\2"+ - "\2\2\u029e\u02a0\3\2\2\2\u029f\u029d\3\2\2\2\u02a0\u02a1\7)\2\2\u02a1"+ - "\u00ae\3\2\2\2\u02a2\u02a4\5\u00bf`\2\u02a3\u02a2\3\2\2\2\u02a4\u02a5"+ - "\3\2\2\2\u02a5\u02a3\3\2\2\2\u02a5\u02a6\3\2\2\2\u02a6\u00b0\3\2\2\2\u02a7"+ - "\u02a9\5\u00bf`\2\u02a8\u02a7\3\2\2\2\u02a9\u02aa\3\2\2\2\u02aa\u02a8"+ - "\3\2\2\2\u02aa\u02ab\3\2\2\2\u02ab\u02ac\3\2\2\2\u02ac\u02b0\5\u00a9U"+ - "\2\u02ad\u02af\5\u00bf`\2\u02ae\u02ad\3\2\2\2\u02af\u02b2\3\2\2\2\u02b0"+ - "\u02ae\3\2\2\2\u02b0\u02b1\3\2\2\2\u02b1\u02d2\3\2\2\2\u02b2\u02b0\3\2"+ - "\2\2\u02b3\u02b5\5\u00a9U\2\u02b4\u02b6\5\u00bf`\2\u02b5\u02b4\3\2\2\2"+ - "\u02b6\u02b7\3\2\2\2\u02b7\u02b5\3\2\2\2\u02b7\u02b8\3\2\2\2\u02b8\u02d2"+ - "\3\2\2\2\u02b9\u02bb\5\u00bf`\2\u02ba\u02b9\3\2\2\2\u02bb\u02bc\3\2\2"+ - "\2\u02bc\u02ba\3\2\2\2\u02bc\u02bd\3\2\2\2\u02bd\u02c5\3\2\2\2\u02be\u02c2"+ - "\5\u00a9U\2\u02bf\u02c1\5\u00bf`\2\u02c0\u02bf\3\2\2\2\u02c1\u02c4\3\2"+ - "\2\2\u02c2\u02c0\3\2\2\2\u02c2\u02c3\3\2\2\2\u02c3\u02c6\3\2\2\2\u02c4"+ - "\u02c2\3\2\2\2\u02c5\u02be\3\2\2\2\u02c5\u02c6\3\2\2\2\u02c6\u02c7\3\2"+ - "\2\2\u02c7\u02c8\5\u00bd_\2\u02c8\u02d2\3\2\2\2\u02c9\u02cb\5\u00a9U\2"+ - "\u02ca\u02cc\5\u00bf`\2\u02cb\u02ca\3\2\2\2\u02cc\u02cd\3\2\2\2\u02cd"+ - "\u02cb\3\2\2\2\u02cd\u02ce\3\2\2\2\u02ce\u02cf\3\2\2\2\u02cf\u02d0\5\u00bd"+ - "_\2\u02d0\u02d2\3\2\2\2\u02d1\u02a8\3\2\2\2\u02d1\u02b3\3\2\2\2\u02d1"+ - "\u02ba\3\2\2\2\u02d1\u02c9\3\2\2\2\u02d2\u00b2\3\2\2\2\u02d3\u02d6\5\u00c1"+ - "a\2\u02d4\u02d6\7a\2\2\u02d5\u02d3\3\2\2\2\u02d5\u02d4\3\2\2\2\u02d6\u02dc"+ - "\3\2\2\2\u02d7\u02db\5\u00c1a\2\u02d8\u02db\5\u00bf`\2\u02d9\u02db\t\3"+ - "\2\2\u02da\u02d7\3\2\2\2\u02da\u02d8\3\2\2\2\u02da\u02d9\3\2\2\2\u02db"+ - "\u02de\3\2\2\2\u02dc\u02da\3\2\2\2\u02dc\u02dd\3\2\2\2\u02dd\u00b4\3\2"+ - "\2\2\u02de\u02dc\3\2\2\2\u02df\u02e3\5\u00bf`\2\u02e0\u02e4\5\u00c1a\2"+ - "\u02e1\u02e4\5\u00bf`\2\u02e2\u02e4\t\4\2\2\u02e3\u02e0\3\2\2\2\u02e3"+ - "\u02e1\3\2\2\2\u02e3\u02e2\3\2\2\2\u02e4\u02e5\3\2\2\2\u02e5\u02e3\3\2"+ - "\2\2\u02e5\u02e6\3\2\2\2\u02e6\u00b6\3\2\2\2\u02e7\u02ec\5\u00c1a\2\u02e8"+ - "\u02ec\5\u00bf`\2\u02e9\u02ec\t\3\2\2\u02ea\u02ec\5\u00a1Q\2\u02eb\u02e7"+ - "\3\2\2\2\u02eb\u02e8\3\2\2\2\u02eb\u02e9\3\2\2\2\u02eb\u02ea\3\2\2\2\u02ec"+ - "\u02ed\3\2\2\2\u02ed\u02eb\3\2\2\2\u02ed\u02ee\3\2\2\2\u02ee\u00b8\3\2"+ - "\2\2\u02ef\u02f5\7$\2\2\u02f0\u02f4\n\5\2\2\u02f1\u02f2\7$\2\2\u02f2\u02f4"+ - "\7$\2\2\u02f3\u02f0\3\2\2\2\u02f3\u02f1\3\2\2\2\u02f4\u02f7\3\2\2\2\u02f5"+ - "\u02f3\3\2\2\2\u02f5\u02f6\3\2\2\2\u02f6\u02f8\3\2\2\2\u02f7\u02f5\3\2"+ - "\2\2\u02f8\u02f9\7$\2\2\u02f9\u00ba\3\2\2\2\u02fa\u0300\7b\2\2\u02fb\u02ff"+ - "\n\6\2\2\u02fc\u02fd\7b\2\2\u02fd\u02ff\7b\2\2\u02fe\u02fb\3\2\2\2\u02fe"+ - "\u02fc\3\2\2\2\u02ff\u0302\3\2\2\2\u0300\u02fe\3\2\2\2\u0300\u0301\3\2"+ - "\2\2\u0301\u0303\3\2\2\2\u0302\u0300\3\2\2\2\u0303\u0304\7b\2\2\u0304"+ - "\u00bc\3\2\2\2\u0305\u0307\7G\2\2\u0306\u0308\t\7\2\2\u0307\u0306\3\2"+ - "\2\2\u0307\u0308\3\2\2\2\u0308\u030a\3\2\2\2\u0309\u030b\5\u00bf`\2\u030a"+ - "\u0309\3\2\2\2\u030b\u030c\3\2\2\2\u030c\u030a\3\2\2\2\u030c\u030d\3\2"+ - "\2\2\u030d\u00be\3\2\2\2\u030e\u030f\t\b\2\2\u030f\u00c0\3\2\2\2\u0310"+ - "\u0311\t\t\2\2\u0311\u00c2\3\2\2\2\u0312\u0313\7/\2\2\u0313\u0314\7/\2"+ - "\2\u0314\u0318\3\2\2\2\u0315\u0317\n\n\2\2\u0316\u0315\3\2\2\2\u0317\u031a"+ - "\3\2\2\2\u0318\u0316\3\2\2\2\u0318\u0319\3\2\2\2\u0319\u031c\3\2\2\2\u031a"+ - "\u0318\3\2\2\2\u031b\u031d\7\17\2\2\u031c\u031b\3\2\2\2\u031c\u031d\3"+ - "\2\2\2\u031d\u031f\3\2\2\2\u031e\u0320\7\f\2\2\u031f\u031e\3\2\2\2\u031f"+ - "\u0320\3\2\2\2\u0320\u0321\3\2\2\2\u0321\u0322\bb\2\2\u0322\u00c4\3\2"+ - "\2\2\u0323\u0324\7\61\2\2\u0324\u0325\7,\2\2\u0325\u032a\3\2\2\2\u0326"+ - "\u0329\5\u00c5c\2\u0327\u0329\13\2\2\2\u0328\u0326\3\2\2\2\u0328\u0327"+ - "\3\2\2\2\u0329\u032c\3\2\2\2\u032a\u032b\3\2\2\2\u032a\u0328\3\2\2\2\u032b"+ - "\u032d\3\2\2\2\u032c\u032a\3\2\2\2\u032d\u032e\7,\2\2\u032e\u032f\7\61"+ - "\2\2\u032f\u0330\3\2\2\2\u0330\u0331\bc\2\2\u0331\u00c6\3\2\2\2\u0332"+ - "\u0334\t\13\2\2\u0333\u0332\3\2\2\2\u0334\u0335\3\2\2\2\u0335\u0333\3"+ - "\2\2\2\u0335\u0336\3\2\2\2\u0336\u0337\3\2\2\2\u0337\u0338\bd\2\2\u0338"+ - "\u00c8\3\2\2\2\u0339\u033a\13\2\2\2\u033a\u00ca\3\2\2\2\"\2\u027a\u029b"+ - "\u029d\u02a5\u02aa\u02b0\u02b7\u02bc\u02c2\u02c5\u02cd\u02d1\u02d5\u02da"+ - "\u02dc\u02e3\u02e5\u02eb\u02ed\u02f3\u02f5\u02fe\u0300\u0307\u030c\u0318"+ - "\u031c\u031f\u0328\u032a\u0335\3\2\3\2"; + "\3\2\2\2\2\u00bd\3\2\2\2\2\u00bf\3\2\2\2\2\u00c1\3\2\2\2\2\u00c3\3\2\2"+ + "\2\2\u00c5\3\2\2\2\2\u00c7\3\2\2\2\2\u00c9\3\2\2\2\2\u00cb\3\2\2\2\2\u00d3"+ + "\3\2\2\2\2\u00d5\3\2\2\2\2\u00d7\3\2\2\2\2\u00d9\3\2\2\2\3\u00db\3\2\2"+ + "\2\5\u00dd\3\2\2\2\7\u00df\3\2\2\2\t\u00e1\3\2\2\2\13\u00e3\3\2\2\2\r"+ + "\u00e7\3\2\2\2\17\u00ef\3\2\2\2\21\u00f8\3\2\2\2\23\u00fc\3\2\2\2\25\u0100"+ + "\3\2\2\2\27\u0103\3\2\2\2\31\u0107\3\2\2\2\33\u010f\3\2\2\2\35\u0112\3"+ + "\2\2\2\37\u0117\3\2\2\2!\u011f\3\2\2\2#\u0128\3\2\2\2%\u0130\3\2\2\2\'"+ + "\u0136\3\2\2\2)\u013b\3\2\2\2+\u0144\3\2\2\2-\u014d\3\2\2\2/\u0154\3\2"+ + "\2\2\61\u015f\3\2\2\2\63\u0166\3\2\2\2\65\u016e\3\2\2\2\67\u0176\3\2\2"+ + "\29\u017c\3\2\2\2;\u0183\3\2\2\2=\u0188\3\2\2\2?\u018d\3\2\2\2A\u0197"+ + "\3\2\2\2C\u01a0\3\2\2\2E\u01a6\3\2\2\2G\u01ad\3\2\2\2I\u01b0\3\2\2\2K"+ + "\u01b6\3\2\2\2M\u01b9\3\2\2\2O\u01be\3\2\2\2Q\u01c3\3\2\2\2S\u01c8\3\2"+ + "\2\2U\u01ce\3\2\2\2W\u01d5\3\2\2\2Y\u01db\3\2\2\2[\u01e3\3\2\2\2]\u01e7"+ + "\3\2\2\2_\u01ec\3\2\2\2a\u01ef\3\2\2\2c\u01f9\3\2\2\2e\u01fc\3\2\2\2g"+ + "\u0202\3\2\2\2i\u0208\3\2\2\2k\u020f\3\2\2\2m\u0218\3\2\2\2o\u021d\3\2"+ + "\2\2q\u0223\3\2\2\2s\u0229\3\2\2\2u\u022f\3\2\2\2w\u0237\3\2\2\2y\u023e"+ + "\3\2\2\2{\u0243\3\2\2\2}\u0247\3\2\2\2\177\u024d\3\2\2\2\u0081\u0254\3"+ + "\2\2\2\u0083\u0259\3\2\2\2\u0085\u025e\3\2\2\2\u0087\u0263\3\2\2\2\u0089"+ + "\u0269\3\2\2\2\u008b\u026f\3\2\2\2\u008d\u0276\3\2\2\2\u008f\u027c\3\2"+ + "\2\2\u0091\u0281\3\2\2\2\u0093\u0289\3\2\2\2\u0095\u028d\3\2\2\2\u0097"+ + "\u0294\3\2\2\2\u0099\u0297\3\2\2\2\u009b\u029a\3\2\2\2\u009d\u029e\3\2"+ + "\2\2\u009f\u02a4\3\2\2\2\u00a1\u02a6\3\2\2\2\u00a3\u02af\3\2\2\2\u00a5"+ + "\u02b1\3\2\2\2\u00a7\u02b3\3\2\2\2\u00a9\u02b6\3\2\2\2\u00ab\u02b8\3\2"+ + "\2\2\u00ad\u02bb\3\2\2\2\u00af\u02bd\3\2\2\2\u00b1\u02bf\3\2\2\2\u00b3"+ + "\u02c1\3\2\2\2\u00b5\u02c3\3\2\2\2\u00b7\u02c5\3\2\2\2\u00b9\u02c8\3\2"+ + "\2\2\u00bb\u02ca\3\2\2\2\u00bd\u02cc\3\2\2\2\u00bf\u02d8\3\2\2\2\u00c1"+ + "\u0306\3\2\2\2\u00c3\u030a\3\2\2\2\u00c5\u0314\3\2\2\2\u00c7\u0320\3\2"+ + "\2\2\u00c9\u0324\3\2\2\2\u00cb\u032f\3\2\2\2\u00cd\u033a\3\2\2\2\u00cf"+ + "\u0343\3\2\2\2\u00d1\u0345\3\2\2\2\u00d3\u0347\3\2\2\2\u00d5\u0358\3\2"+ + "\2\2\u00d7\u0368\3\2\2\2\u00d9\u036e\3\2\2\2\u00db\u00dc\7*\2\2\u00dc"+ + "\4\3\2\2\2\u00dd\u00de\7+\2\2\u00de\6\3\2\2\2\u00df\u00e0\7.\2\2\u00e0"+ + "\b\3\2\2\2\u00e1\u00e2\7<\2\2\u00e2\n\3\2\2\2\u00e3\u00e4\7C\2\2\u00e4"+ + "\u00e5\7N\2\2\u00e5\u00e6\7N\2\2\u00e6\f\3\2\2\2\u00e7\u00e8\7C\2\2\u00e8"+ + "\u00e9\7P\2\2\u00e9\u00ea\7C\2\2\u00ea\u00eb\7N\2\2\u00eb\u00ec\7[\2\2"+ + "\u00ec\u00ed\7\\\2\2\u00ed\u00ee\7G\2\2\u00ee\16\3\2\2\2\u00ef\u00f0\7"+ + "C\2\2\u00f0\u00f1\7P\2\2\u00f1\u00f2\7C\2\2\u00f2\u00f3\7N\2\2\u00f3\u00f4"+ + "\7[\2\2\u00f4\u00f5\7\\\2\2\u00f5\u00f6\7G\2\2\u00f6\u00f7\7F\2\2\u00f7"+ + "\20\3\2\2\2\u00f8\u00f9\7C\2\2\u00f9\u00fa\7P\2\2\u00fa\u00fb\7F\2\2\u00fb"+ + "\22\3\2\2\2\u00fc\u00fd\7C\2\2\u00fd\u00fe\7P\2\2\u00fe\u00ff\7[\2\2\u00ff"+ + "\24\3\2\2\2\u0100\u0101\7C\2\2\u0101\u0102\7U\2\2\u0102\26\3\2\2\2\u0103"+ + "\u0104\7C\2\2\u0104\u0105\7U\2\2\u0105\u0106\7E\2\2\u0106\30\3\2\2\2\u0107"+ + "\u0108\7D\2\2\u0108\u0109\7G\2\2\u0109\u010a\7V\2\2\u010a\u010b\7Y\2\2"+ + "\u010b\u010c\7G\2\2\u010c\u010d\7G\2\2\u010d\u010e\7P\2\2\u010e\32\3\2"+ + "\2\2\u010f\u0110\7D\2\2\u0110\u0111\7[\2\2\u0111\34\3\2\2\2\u0112\u0113"+ + "\7E\2\2\u0113\u0114\7C\2\2\u0114\u0115\7U\2\2\u0115\u0116\7V\2\2\u0116"+ + "\36\3\2\2\2\u0117\u0118\7E\2\2\u0118\u0119\7C\2\2\u0119\u011a\7V\2\2\u011a"+ + "\u011b\7C\2\2\u011b\u011c\7N\2\2\u011c\u011d\7Q\2\2\u011d\u011e\7I\2\2"+ + "\u011e \3\2\2\2\u011f\u0120\7E\2\2\u0120\u0121\7C\2\2\u0121\u0122\7V\2"+ + "\2\u0122\u0123\7C\2\2\u0123\u0124\7N\2\2\u0124\u0125\7Q\2\2\u0125\u0126"+ + "\7I\2\2\u0126\u0127\7U\2\2\u0127\"\3\2\2\2\u0128\u0129\7E\2\2\u0129\u012a"+ + "\7Q\2\2\u012a\u012b\7N\2\2\u012b\u012c\7W\2\2\u012c\u012d\7O\2\2\u012d"+ + "\u012e\7P\2\2\u012e\u012f\7U\2\2\u012f$\3\2\2\2\u0130\u0131\7F\2\2\u0131"+ + "\u0132\7G\2\2\u0132\u0133\7D\2\2\u0133\u0134\7W\2\2\u0134\u0135\7I\2\2"+ + "\u0135&\3\2\2\2\u0136\u0137\7F\2\2\u0137\u0138\7G\2\2\u0138\u0139\7U\2"+ + "\2\u0139\u013a\7E\2\2\u013a(\3\2\2\2\u013b\u013c\7F\2\2\u013c\u013d\7"+ + "G\2\2\u013d\u013e\7U\2\2\u013e\u013f\7E\2\2\u013f\u0140\7T\2\2\u0140\u0141"+ + "\7K\2\2\u0141\u0142\7D\2\2\u0142\u0143\7G\2\2\u0143*\3\2\2\2\u0144\u0145"+ + "\7F\2\2\u0145\u0146\7K\2\2\u0146\u0147\7U\2\2\u0147\u0148\7V\2\2\u0148"+ + "\u0149\7K\2\2\u0149\u014a\7P\2\2\u014a\u014b\7E\2\2\u014b\u014c\7V\2\2"+ + "\u014c,\3\2\2\2\u014d\u014e\7G\2\2\u014e\u014f\7U\2\2\u014f\u0150\7E\2"+ + "\2\u0150\u0151\7C\2\2\u0151\u0152\7R\2\2\u0152\u0153\7G\2\2\u0153.\3\2"+ + "\2\2\u0154\u0155\7G\2\2\u0155\u0156\7Z\2\2\u0156\u0157\7G\2\2\u0157\u0158"+ + "\7E\2\2\u0158\u0159\7W\2\2\u0159\u015a\7V\2\2\u015a\u015b\7C\2\2\u015b"+ + "\u015c\7D\2\2\u015c\u015d\7N\2\2\u015d\u015e\7G\2\2\u015e\60\3\2\2\2\u015f"+ + "\u0160\7G\2\2\u0160\u0161\7Z\2\2\u0161\u0162\7K\2\2\u0162\u0163\7U\2\2"+ + "\u0163\u0164\7V\2\2\u0164\u0165\7U\2\2\u0165\62\3\2\2\2\u0166\u0167\7"+ + "G\2\2\u0167\u0168\7Z\2\2\u0168\u0169\7R\2\2\u0169\u016a\7N\2\2\u016a\u016b"+ + "\7C\2\2\u016b\u016c\7K\2\2\u016c\u016d\7P\2\2\u016d\64\3\2\2\2\u016e\u016f"+ + "\7G\2\2\u016f\u0170\7Z\2\2\u0170\u0171\7V\2\2\u0171\u0172\7T\2\2\u0172"+ + "\u0173\7C\2\2\u0173\u0174\7E\2\2\u0174\u0175\7V\2\2\u0175\66\3\2\2\2\u0176"+ + "\u0177\7H\2\2\u0177\u0178\7C\2\2\u0178\u0179\7N\2\2\u0179\u017a\7U\2\2"+ + "\u017a\u017b\7G\2\2\u017b8\3\2\2\2\u017c\u017d\7H\2\2\u017d\u017e\7Q\2"+ + "\2\u017e\u017f\7T\2\2\u017f\u0180\7O\2\2\u0180\u0181\7C\2\2\u0181\u0182"+ + "\7V\2\2\u0182:\3\2\2\2\u0183\u0184\7H\2\2\u0184\u0185\7T\2\2\u0185\u0186"+ + "\7Q\2\2\u0186\u0187\7O\2\2\u0187<\3\2\2\2\u0188\u0189\7H\2\2\u0189\u018a"+ + "\7W\2\2\u018a\u018b\7N\2\2\u018b\u018c\7N\2\2\u018c>\3\2\2\2\u018d\u018e"+ + "\7H\2\2\u018e\u018f\7W\2\2\u018f\u0190\7P\2\2\u0190\u0191\7E\2\2\u0191"+ + "\u0192\7V\2\2\u0192\u0193\7K\2\2\u0193\u0194\7Q\2\2\u0194\u0195\7P\2\2"+ + "\u0195\u0196\7U\2\2\u0196@\3\2\2\2\u0197\u0198\7I\2\2\u0198\u0199\7T\2"+ + "\2\u0199\u019a\7C\2\2\u019a\u019b\7R\2\2\u019b\u019c\7J\2\2\u019c\u019d"+ + "\7X\2\2\u019d\u019e\7K\2\2\u019e\u019f\7\\\2\2\u019fB\3\2\2\2\u01a0\u01a1"+ + "\7I\2\2\u01a1\u01a2\7T\2\2\u01a2\u01a3\7Q\2\2\u01a3\u01a4\7W\2\2\u01a4"+ + "\u01a5\7R\2\2\u01a5D\3\2\2\2\u01a6\u01a7\7J\2\2\u01a7\u01a8\7C\2\2\u01a8"+ + "\u01a9\7X\2\2\u01a9\u01aa\7K\2\2\u01aa\u01ab\7P\2\2\u01ab\u01ac\7I\2\2"+ + "\u01acF\3\2\2\2\u01ad\u01ae\7K\2\2\u01ae\u01af\7P\2\2\u01afH\3\2\2\2\u01b0"+ + "\u01b1\7K\2\2\u01b1\u01b2\7P\2\2\u01b2\u01b3\7P\2\2\u01b3\u01b4\7G\2\2"+ + "\u01b4\u01b5\7T\2\2\u01b5J\3\2\2\2\u01b6\u01b7\7K\2\2\u01b7\u01b8\7U\2"+ + "\2\u01b8L\3\2\2\2\u01b9\u01ba\7L\2\2\u01ba\u01bb\7Q\2\2\u01bb\u01bc\7"+ + "K\2\2\u01bc\u01bd\7P\2\2\u01bdN\3\2\2\2\u01be\u01bf\7N\2\2\u01bf\u01c0"+ + "\7G\2\2\u01c0\u01c1\7H\2\2\u01c1\u01c2\7V\2\2\u01c2P\3\2\2\2\u01c3\u01c4"+ + "\7N\2\2\u01c4\u01c5\7K\2\2\u01c5\u01c6\7M\2\2\u01c6\u01c7\7G\2\2\u01c7"+ + "R\3\2\2\2\u01c8\u01c9\7N\2\2\u01c9\u01ca\7K\2\2\u01ca\u01cb\7O\2\2\u01cb"+ + "\u01cc\7K\2\2\u01cc\u01cd\7V\2\2\u01cdT\3\2\2\2\u01ce\u01cf\7O\2\2\u01cf"+ + "\u01d0\7C\2\2\u01d0\u01d1\7R\2\2\u01d1\u01d2\7R\2\2\u01d2\u01d3\7G\2\2"+ + "\u01d3\u01d4\7F\2\2\u01d4V\3\2\2\2\u01d5\u01d6\7O\2\2\u01d6\u01d7\7C\2"+ + "\2\u01d7\u01d8\7V\2\2\u01d8\u01d9\7E\2\2\u01d9\u01da\7J\2\2\u01daX\3\2"+ + "\2\2\u01db\u01dc\7P\2\2\u01dc\u01dd\7C\2\2\u01dd\u01de\7V\2\2\u01de\u01df"+ + "\7W\2\2\u01df\u01e0\7T\2\2\u01e0\u01e1\7C\2\2\u01e1\u01e2\7N\2\2\u01e2"+ + "Z\3\2\2\2\u01e3\u01e4\7P\2\2\u01e4\u01e5\7Q\2\2\u01e5\u01e6\7V\2\2\u01e6"+ + "\\\3\2\2\2\u01e7\u01e8\7P\2\2\u01e8\u01e9\7W\2\2\u01e9\u01ea\7N\2\2\u01ea"+ + "\u01eb\7N\2\2\u01eb^\3\2\2\2\u01ec\u01ed\7Q\2\2\u01ed\u01ee\7P\2\2\u01ee"+ + "`\3\2\2\2\u01ef\u01f0\7Q\2\2\u01f0\u01f1\7R\2\2\u01f1\u01f2\7V\2\2\u01f2"+ + "\u01f3\7K\2\2\u01f3\u01f4\7O\2\2\u01f4\u01f5\7K\2\2\u01f5\u01f6\7\\\2"+ + "\2\u01f6\u01f7\7G\2\2\u01f7\u01f8\7F\2\2\u01f8b\3\2\2\2\u01f9\u01fa\7"+ + "Q\2\2\u01fa\u01fb\7T\2\2\u01fbd\3\2\2\2\u01fc\u01fd\7Q\2\2\u01fd\u01fe"+ + "\7T\2\2\u01fe\u01ff\7F\2\2\u01ff\u0200\7G\2\2\u0200\u0201\7T\2\2\u0201"+ + "f\3\2\2\2\u0202\u0203\7Q\2\2\u0203\u0204\7W\2\2\u0204\u0205\7V\2\2\u0205"+ + "\u0206\7G\2\2\u0206\u0207\7T\2\2\u0207h\3\2\2\2\u0208\u0209\7R\2\2\u0209"+ + "\u020a\7C\2\2\u020a\u020b\7T\2\2\u020b\u020c\7U\2\2\u020c\u020d\7G\2\2"+ + "\u020d\u020e\7F\2\2\u020ej\3\2\2\2\u020f\u0210\7R\2\2\u0210\u0211\7J\2"+ + "\2\u0211\u0212\7[\2\2\u0212\u0213\7U\2\2\u0213\u0214\7K\2\2\u0214\u0215"+ + "\7E\2\2\u0215\u0216\7C\2\2\u0216\u0217\7N\2\2\u0217l\3\2\2\2\u0218\u0219"+ + "\7R\2\2\u0219\u021a\7N\2\2\u021a\u021b\7C\2\2\u021b\u021c\7P\2\2\u021c"+ + "n\3\2\2\2\u021d\u021e\7T\2\2\u021e\u021f\7K\2\2\u021f\u0220\7I\2\2\u0220"+ + "\u0221\7J\2\2\u0221\u0222\7V\2\2\u0222p\3\2\2\2\u0223\u0224\7T\2\2\u0224"+ + "\u0225\7N\2\2\u0225\u0226\7K\2\2\u0226\u0227\7M\2\2\u0227\u0228\7G\2\2"+ + "\u0228r\3\2\2\2\u0229\u022a\7S\2\2\u022a\u022b\7W\2\2\u022b\u022c\7G\2"+ + "\2\u022c\u022d\7T\2\2\u022d\u022e\7[\2\2\u022et\3\2\2\2\u022f\u0230\7"+ + "U\2\2\u0230\u0231\7E\2\2\u0231\u0232\7J\2\2\u0232\u0233\7G\2\2\u0233\u0234"+ + "\7O\2\2\u0234\u0235\7C\2\2\u0235\u0236\7U\2\2\u0236v\3\2\2\2\u0237\u0238"+ + "\7U\2\2\u0238\u0239\7G\2\2\u0239\u023a\7N\2\2\u023a\u023b\7G\2\2\u023b"+ + "\u023c\7E\2\2\u023c\u023d\7V\2\2\u023dx\3\2\2\2\u023e\u023f\7U\2\2\u023f"+ + "\u0240\7J\2\2\u0240\u0241\7Q\2\2\u0241\u0242\7Y\2\2\u0242z\3\2\2\2\u0243"+ + "\u0244\7U\2\2\u0244\u0245\7[\2\2\u0245\u0246\7U\2\2\u0246|\3\2\2\2\u0247"+ + "\u0248\7V\2\2\u0248\u0249\7C\2\2\u0249\u024a\7D\2\2\u024a\u024b\7N\2\2"+ + "\u024b\u024c\7G\2\2\u024c~\3\2\2\2\u024d\u024e\7V\2\2\u024e\u024f\7C\2"+ + "\2\u024f\u0250\7D\2\2\u0250\u0251\7N\2\2\u0251\u0252\7G\2\2\u0252\u0253"+ + "\7U\2\2\u0253\u0080\3\2\2\2\u0254\u0255\7V\2\2\u0255\u0256\7G\2\2\u0256"+ + "\u0257\7Z\2\2\u0257\u0258\7V\2\2\u0258\u0082\3\2\2\2\u0259\u025a\7V\2"+ + "\2\u025a\u025b\7T\2\2\u025b\u025c\7W\2\2\u025c\u025d\7G\2\2\u025d\u0084"+ + "\3\2\2\2\u025e\u025f\7V\2\2\u025f\u0260\7[\2\2\u0260\u0261\7R\2\2\u0261"+ + "\u0262\7G\2\2\u0262\u0086\3\2\2\2\u0263\u0264\7V\2\2\u0264\u0265\7[\2"+ + "\2\u0265\u0266\7R\2\2\u0266\u0267\7G\2\2\u0267\u0268\7U\2\2\u0268\u0088"+ + "\3\2\2\2\u0269\u026a\7W\2\2\u026a\u026b\7U\2\2\u026b\u026c\7K\2\2\u026c"+ + "\u026d\7P\2\2\u026d\u026e\7I\2\2\u026e\u008a\3\2\2\2\u026f\u0270\7X\2"+ + "\2\u0270\u0271\7G\2\2\u0271\u0272\7T\2\2\u0272\u0273\7K\2\2\u0273\u0274"+ + "\7H\2\2\u0274\u0275\7[\2\2\u0275\u008c\3\2\2\2\u0276\u0277\7Y\2\2\u0277"+ + "\u0278\7J\2\2\u0278\u0279\7G\2\2\u0279\u027a\7T\2\2\u027a\u027b\7G\2\2"+ + "\u027b\u008e\3\2\2\2\u027c\u027d\7Y\2\2\u027d\u027e\7K\2\2\u027e\u027f"+ + "\7V\2\2\u027f\u0280\7J\2\2\u0280\u0090\3\2\2\2\u0281\u0282\7}\2\2\u0282"+ + "\u0283\7G\2\2\u0283\u0284\7U\2\2\u0284\u0285\7E\2\2\u0285\u0286\7C\2\2"+ + "\u0286\u0287\7R\2\2\u0287\u0288\7G\2\2\u0288\u0092\3\2\2\2\u0289\u028a"+ + "\7}\2\2\u028a\u028b\7H\2\2\u028b\u028c\7P\2\2\u028c\u0094\3\2\2\2\u028d"+ + "\u028e\7}\2\2\u028e\u028f\7N\2\2\u028f\u0290\7K\2\2\u0290\u0291\7O\2\2"+ + "\u0291\u0292\7K\2\2\u0292\u0293\7V\2\2\u0293\u0096\3\2\2\2\u0294\u0295"+ + "\7}\2\2\u0295\u0296\7F\2\2\u0296\u0098\3\2\2\2\u0297\u0298\7}\2\2\u0298"+ + "\u0299\7V\2\2\u0299\u009a\3\2\2\2\u029a\u029b\7}\2\2\u029b\u029c\7V\2"+ + "\2\u029c\u029d\7U\2\2\u029d\u009c\3\2\2\2\u029e\u029f\7}\2\2\u029f\u02a0"+ + "\7I\2\2\u02a0\u02a1\7W\2\2\u02a1\u02a2\7K\2\2\u02a2\u02a3\7F\2\2\u02a3"+ + "\u009e\3\2\2\2\u02a4\u02a5\7\177\2\2\u02a5\u00a0\3\2\2\2\u02a6\u02a7\7"+ + "?\2\2\u02a7\u00a2\3\2\2\2\u02a8\u02a9\7>\2\2\u02a9\u02b0\7@\2\2\u02aa"+ + "\u02ab\7#\2\2\u02ab\u02b0\7?\2\2\u02ac\u02ad\7>\2\2\u02ad\u02ae\7?\2\2"+ + "\u02ae\u02b0\7@\2\2\u02af\u02a8\3\2\2\2\u02af\u02aa\3\2\2\2\u02af\u02ac"+ + "\3\2\2\2\u02b0\u00a4\3\2\2\2\u02b1\u02b2\7>\2\2\u02b2\u00a6\3\2\2\2\u02b3"+ + "\u02b4\7>\2\2\u02b4\u02b5\7?\2\2\u02b5\u00a8\3\2\2\2\u02b6\u02b7\7@\2"+ + "\2\u02b7\u00aa\3\2\2\2\u02b8\u02b9\7@\2\2\u02b9\u02ba\7?\2\2\u02ba\u00ac"+ + "\3\2\2\2\u02bb\u02bc\7-\2\2\u02bc\u00ae\3\2\2\2\u02bd\u02be\7/\2\2\u02be"+ + "\u00b0\3\2\2\2\u02bf\u02c0\7,\2\2\u02c0\u00b2\3\2\2\2\u02c1\u02c2\7\61"+ + "\2\2\u02c2\u00b4\3\2\2\2\u02c3\u02c4\7\'\2\2\u02c4\u00b6\3\2\2\2\u02c5"+ + "\u02c6\7~\2\2\u02c6\u02c7\7~\2\2\u02c7\u00b8\3\2\2\2\u02c8\u02c9\7\60"+ + "\2\2\u02c9\u00ba\3\2\2\2\u02ca\u02cb\7A\2\2\u02cb\u00bc\3\2\2\2\u02cc"+ + "\u02d2\7)\2\2\u02cd\u02d1\n\2\2\2\u02ce\u02cf\7)\2\2\u02cf\u02d1\7)\2"+ + "\2\u02d0\u02cd\3\2\2\2\u02d0\u02ce\3\2\2\2\u02d1\u02d4\3\2\2\2\u02d2\u02d0"+ + "\3\2\2\2\u02d2\u02d3\3\2\2\2\u02d3\u02d5\3\2\2\2\u02d4\u02d2\3\2\2\2\u02d5"+ + "\u02d6\7)\2\2\u02d6\u00be\3\2\2\2\u02d7\u02d9\5\u00cfh\2\u02d8\u02d7\3"+ + "\2\2\2\u02d9\u02da\3\2\2\2\u02da\u02d8\3\2\2\2\u02da\u02db\3\2\2\2\u02db"+ + "\u00c0\3\2\2\2\u02dc\u02de\5\u00cfh\2\u02dd\u02dc\3\2\2\2\u02de\u02df"+ + "\3\2\2\2\u02df\u02dd\3\2\2\2\u02df\u02e0\3\2\2\2\u02e0\u02e1\3\2\2\2\u02e1"+ + "\u02e5\5\u00b9]\2\u02e2\u02e4\5\u00cfh\2\u02e3\u02e2\3\2\2\2\u02e4\u02e7"+ + "\3\2\2\2\u02e5\u02e3\3\2\2\2\u02e5\u02e6\3\2\2\2\u02e6\u0307\3\2\2\2\u02e7"+ + "\u02e5\3\2\2\2\u02e8\u02ea\5\u00b9]\2\u02e9\u02eb\5\u00cfh\2\u02ea\u02e9"+ + "\3\2\2\2\u02eb\u02ec\3\2\2\2\u02ec\u02ea\3\2\2\2\u02ec\u02ed\3\2\2\2\u02ed"+ + "\u0307\3\2\2\2\u02ee\u02f0\5\u00cfh\2\u02ef\u02ee\3\2\2\2\u02f0\u02f1"+ + "\3\2\2\2\u02f1\u02ef\3\2\2\2\u02f1\u02f2\3\2\2\2\u02f2\u02fa\3\2\2\2\u02f3"+ + "\u02f7\5\u00b9]\2\u02f4\u02f6\5\u00cfh\2\u02f5\u02f4\3\2\2\2\u02f6\u02f9"+ + "\3\2\2\2\u02f7\u02f5\3\2\2\2\u02f7\u02f8\3\2\2\2\u02f8\u02fb\3\2\2\2\u02f9"+ + "\u02f7\3\2\2\2\u02fa\u02f3\3\2\2\2\u02fa\u02fb\3\2\2\2\u02fb\u02fc\3\2"+ + "\2\2\u02fc\u02fd\5\u00cdg\2\u02fd\u0307\3\2\2\2\u02fe\u0300\5\u00b9]\2"+ + "\u02ff\u0301\5\u00cfh\2\u0300\u02ff\3\2\2\2\u0301\u0302\3\2\2\2\u0302"+ + "\u0300\3\2\2\2\u0302\u0303\3\2\2\2\u0303\u0304\3\2\2\2\u0304\u0305\5\u00cd"+ + "g\2\u0305\u0307\3\2\2\2\u0306\u02dd\3\2\2\2\u0306\u02e8\3\2\2\2\u0306"+ + "\u02ef\3\2\2\2\u0306\u02fe\3\2\2\2\u0307\u00c2\3\2\2\2\u0308\u030b\5\u00d1"+ + "i\2\u0309\u030b\7a\2\2\u030a\u0308\3\2\2\2\u030a\u0309\3\2\2\2\u030b\u0311"+ + "\3\2\2\2\u030c\u0310\5\u00d1i\2\u030d\u0310\5\u00cfh\2\u030e\u0310\t\3"+ + "\2\2\u030f\u030c\3\2\2\2\u030f\u030d\3\2\2\2\u030f\u030e\3\2\2\2\u0310"+ + "\u0313\3\2\2\2\u0311\u030f\3\2\2\2\u0311\u0312\3\2\2\2\u0312\u00c4\3\2"+ + "\2\2\u0313\u0311\3\2\2\2\u0314\u0318\5\u00cfh\2\u0315\u0319\5\u00d1i\2"+ + "\u0316\u0319\5\u00cfh\2\u0317\u0319\t\4\2\2\u0318\u0315\3\2\2\2\u0318"+ + "\u0316\3\2\2\2\u0318\u0317\3\2\2\2\u0319\u031a\3\2\2\2\u031a\u0318\3\2"+ + "\2\2\u031a\u031b\3\2\2\2\u031b\u00c6\3\2\2\2\u031c\u0321\5\u00d1i\2\u031d"+ + "\u0321\5\u00cfh\2\u031e\u0321\t\3\2\2\u031f\u0321\5\u00b1Y\2\u0320\u031c"+ + "\3\2\2\2\u0320\u031d\3\2\2\2\u0320\u031e\3\2\2\2\u0320\u031f\3\2\2\2\u0321"+ + "\u0322\3\2\2\2\u0322\u0320\3\2\2\2\u0322\u0323\3\2\2\2\u0323\u00c8\3\2"+ + "\2\2\u0324\u032a\7$\2\2\u0325\u0329\n\5\2\2\u0326\u0327\7$\2\2\u0327\u0329"+ + "\7$\2\2\u0328\u0325\3\2\2\2\u0328\u0326\3\2\2\2\u0329\u032c\3\2\2\2\u032a"+ + "\u0328\3\2\2\2\u032a\u032b\3\2\2\2\u032b\u032d\3\2\2\2\u032c\u032a\3\2"+ + "\2\2\u032d\u032e\7$\2\2\u032e\u00ca\3\2\2\2\u032f\u0335\7b\2\2\u0330\u0334"+ + "\n\6\2\2\u0331\u0332\7b\2\2\u0332\u0334\7b\2\2\u0333\u0330\3\2\2\2\u0333"+ + "\u0331\3\2\2\2\u0334\u0337\3\2\2\2\u0335\u0333\3\2\2\2\u0335\u0336\3\2"+ + "\2\2\u0336\u0338\3\2\2\2\u0337\u0335\3\2\2\2\u0338\u0339\7b\2\2\u0339"+ + "\u00cc\3\2\2\2\u033a\u033c\7G\2\2\u033b\u033d\t\7\2\2\u033c\u033b\3\2"+ + "\2\2\u033c\u033d\3\2\2\2\u033d\u033f\3\2\2\2\u033e\u0340\5\u00cfh\2\u033f"+ + "\u033e\3\2\2\2\u0340\u0341\3\2\2\2\u0341\u033f\3\2\2\2\u0341\u0342\3\2"+ + "\2\2\u0342\u00ce\3\2\2\2\u0343\u0344\t\b\2\2\u0344\u00d0\3\2\2\2\u0345"+ + "\u0346\t\t\2\2\u0346\u00d2\3\2\2\2\u0347\u0348\7/\2\2\u0348\u0349\7/\2"+ + "\2\u0349\u034d\3\2\2\2\u034a\u034c\n\n\2\2\u034b\u034a\3\2\2\2\u034c\u034f"+ + "\3\2\2\2\u034d\u034b\3\2\2\2\u034d\u034e\3\2\2\2\u034e\u0351\3\2\2\2\u034f"+ + "\u034d\3\2\2\2\u0350\u0352\7\17\2\2\u0351\u0350\3\2\2\2\u0351\u0352\3"+ + "\2\2\2\u0352\u0354\3\2\2\2\u0353\u0355\7\f\2\2\u0354\u0353\3\2\2\2\u0354"+ + "\u0355\3\2\2\2\u0355\u0356\3\2\2\2\u0356\u0357\bj\2\2\u0357\u00d4\3\2"+ + "\2\2\u0358\u0359\7\61\2\2\u0359\u035a\7,\2\2\u035a\u035f\3\2\2\2\u035b"+ + "\u035e\5\u00d5k\2\u035c\u035e\13\2\2\2\u035d\u035b\3\2\2\2\u035d\u035c"+ + "\3\2\2\2\u035e\u0361\3\2\2\2\u035f\u0360\3\2\2\2\u035f\u035d\3\2\2\2\u0360"+ + "\u0362\3\2\2\2\u0361\u035f\3\2\2\2\u0362\u0363\7,\2\2\u0363\u0364\7\61"+ + "\2\2\u0364\u0365\3\2\2\2\u0365\u0366\bk\2\2\u0366\u00d6\3\2\2\2\u0367"+ + "\u0369\t\13\2\2\u0368\u0367\3\2\2\2\u0369\u036a\3\2\2\2\u036a\u0368\3"+ + "\2\2\2\u036a\u036b\3\2\2\2\u036b\u036c\3\2\2\2\u036c\u036d\bl\2\2\u036d"+ + "\u00d8\3\2\2\2\u036e\u036f\13\2\2\2\u036f\u00da\3\2\2\2\"\2\u02af\u02d0"+ + "\u02d2\u02da\u02df\u02e5\u02ec\u02f1\u02f7\u02fa\u0302\u0306\u030a\u030f"+ + "\u0311\u0318\u031a\u0320\u0322\u0328\u032a\u0333\u0335\u033c\u0341\u034d"+ + "\u0351\u0354\u035d\u035f\u036a\3\2\3\2"; public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray()); static { diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseListener.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseListener.java index 48f6eb4a7c816..70fe777384c85 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseListener.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseListener.java @@ -1,8 +1,3 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ // ANTLR GENERATED CODE: DO NOT EDIT package org.elasticsearch.xpack.sql.parser; import org.antlr.v4.runtime.tree.ParseTreeListener; @@ -196,6 +191,16 @@ interface SqlBaseListener extends ParseTreeListener { * @param ctx the parse tree */ void exitQueryNoWith(SqlBaseParser.QueryNoWithContext ctx); + /** + * Enter a parse tree produced by {@link SqlBaseParser#limitClause}. + * @param ctx the parse tree + */ + void enterLimitClause(SqlBaseParser.LimitClauseContext ctx); + /** + * Exit a parse tree produced by {@link SqlBaseParser#limitClause}. + * @param ctx the parse tree + */ + void exitLimitClause(SqlBaseParser.LimitClauseContext ctx); /** * Enter a parse tree produced by the {@code queryPrimaryDefault} * labeled alternative in {@link SqlBaseParser#queryTerm}. @@ -514,6 +519,16 @@ interface SqlBaseListener extends ParseTreeListener { * @param ctx the parse tree */ void exitPattern(SqlBaseParser.PatternContext ctx); + /** + * Enter a parse tree produced by {@link SqlBaseParser#patternEscape}. + * @param ctx the parse tree + */ + void enterPatternEscape(SqlBaseParser.PatternEscapeContext ctx); + /** + * Exit a parse tree produced by {@link SqlBaseParser#patternEscape}. + * @param ctx the parse tree + */ + void exitPatternEscape(SqlBaseParser.PatternEscapeContext ctx); /** * Enter a parse tree produced by the {@code valueExpressionDefault} * labeled alternative in {@link SqlBaseParser#valueExpression}. @@ -611,17 +626,17 @@ interface SqlBaseListener extends ParseTreeListener { */ void exitStar(SqlBaseParser.StarContext ctx); /** - * Enter a parse tree produced by the {@code functionCall} + * Enter a parse tree produced by the {@code function} * labeled alternative in {@link SqlBaseParser#primaryExpression}. * @param ctx the parse tree */ - void enterFunctionCall(SqlBaseParser.FunctionCallContext ctx); + void enterFunction(SqlBaseParser.FunctionContext ctx); /** - * Exit a parse tree produced by the {@code functionCall} + * Exit a parse tree produced by the {@code function} * labeled alternative in {@link SqlBaseParser#primaryExpression}. * @param ctx the parse tree */ - void exitFunctionCall(SqlBaseParser.FunctionCallContext ctx); + void exitFunction(SqlBaseParser.FunctionContext ctx); /** * Enter a parse tree produced by the {@code subqueryExpression} * labeled alternative in {@link SqlBaseParser#primaryExpression}. @@ -670,6 +685,66 @@ interface SqlBaseListener extends ParseTreeListener { * @param ctx the parse tree */ void exitParenthesizedExpression(SqlBaseParser.ParenthesizedExpressionContext ctx); + /** + * Enter a parse tree produced by {@link SqlBaseParser#castExpression}. + * @param ctx the parse tree + */ + void enterCastExpression(SqlBaseParser.CastExpressionContext ctx); + /** + * Exit a parse tree produced by {@link SqlBaseParser#castExpression}. + * @param ctx the parse tree + */ + void exitCastExpression(SqlBaseParser.CastExpressionContext ctx); + /** + * Enter a parse tree produced by {@link SqlBaseParser#castTemplate}. + * @param ctx the parse tree + */ + void enterCastTemplate(SqlBaseParser.CastTemplateContext ctx); + /** + * Exit a parse tree produced by {@link SqlBaseParser#castTemplate}. + * @param ctx the parse tree + */ + void exitCastTemplate(SqlBaseParser.CastTemplateContext ctx); + /** + * Enter a parse tree produced by {@link SqlBaseParser#extractExpression}. + * @param ctx the parse tree + */ + void enterExtractExpression(SqlBaseParser.ExtractExpressionContext ctx); + /** + * Exit a parse tree produced by {@link SqlBaseParser#extractExpression}. + * @param ctx the parse tree + */ + void exitExtractExpression(SqlBaseParser.ExtractExpressionContext ctx); + /** + * Enter a parse tree produced by {@link SqlBaseParser#extractTemplate}. + * @param ctx the parse tree + */ + void enterExtractTemplate(SqlBaseParser.ExtractTemplateContext ctx); + /** + * Exit a parse tree produced by {@link SqlBaseParser#extractTemplate}. + * @param ctx the parse tree + */ + void exitExtractTemplate(SqlBaseParser.ExtractTemplateContext ctx); + /** + * Enter a parse tree produced by {@link SqlBaseParser#functionExpression}. + * @param ctx the parse tree + */ + void enterFunctionExpression(SqlBaseParser.FunctionExpressionContext ctx); + /** + * Exit a parse tree produced by {@link SqlBaseParser#functionExpression}. + * @param ctx the parse tree + */ + void exitFunctionExpression(SqlBaseParser.FunctionExpressionContext ctx); + /** + * Enter a parse tree produced by {@link SqlBaseParser#functionTemplate}. + * @param ctx the parse tree + */ + void enterFunctionTemplate(SqlBaseParser.FunctionTemplateContext ctx); + /** + * Exit a parse tree produced by {@link SqlBaseParser#functionTemplate}. + * @param ctx the parse tree + */ + void exitFunctionTemplate(SqlBaseParser.FunctionTemplateContext ctx); /** * Enter a parse tree produced by the {@code nullLiteral} * labeled alternative in {@link SqlBaseParser#constant}. @@ -730,6 +805,54 @@ interface SqlBaseListener extends ParseTreeListener { * @param ctx the parse tree */ void exitParamLiteral(SqlBaseParser.ParamLiteralContext ctx); + /** + * Enter a parse tree produced by the {@code dateEscapedLiteral} + * labeled alternative in {@link SqlBaseParser#constant}. + * @param ctx the parse tree + */ + void enterDateEscapedLiteral(SqlBaseParser.DateEscapedLiteralContext ctx); + /** + * Exit a parse tree produced by the {@code dateEscapedLiteral} + * labeled alternative in {@link SqlBaseParser#constant}. + * @param ctx the parse tree + */ + void exitDateEscapedLiteral(SqlBaseParser.DateEscapedLiteralContext ctx); + /** + * Enter a parse tree produced by the {@code timeEscapedLiteral} + * labeled alternative in {@link SqlBaseParser#constant}. + * @param ctx the parse tree + */ + void enterTimeEscapedLiteral(SqlBaseParser.TimeEscapedLiteralContext ctx); + /** + * Exit a parse tree produced by the {@code timeEscapedLiteral} + * labeled alternative in {@link SqlBaseParser#constant}. + * @param ctx the parse tree + */ + void exitTimeEscapedLiteral(SqlBaseParser.TimeEscapedLiteralContext ctx); + /** + * Enter a parse tree produced by the {@code timestampEscapedLiteral} + * labeled alternative in {@link SqlBaseParser#constant}. + * @param ctx the parse tree + */ + void enterTimestampEscapedLiteral(SqlBaseParser.TimestampEscapedLiteralContext ctx); + /** + * Exit a parse tree produced by the {@code timestampEscapedLiteral} + * labeled alternative in {@link SqlBaseParser#constant}. + * @param ctx the parse tree + */ + void exitTimestampEscapedLiteral(SqlBaseParser.TimestampEscapedLiteralContext ctx); + /** + * Enter a parse tree produced by the {@code guidEscapedLiteral} + * labeled alternative in {@link SqlBaseParser#constant}. + * @param ctx the parse tree + */ + void enterGuidEscapedLiteral(SqlBaseParser.GuidEscapedLiteralContext ctx); + /** + * Exit a parse tree produced by the {@code guidEscapedLiteral} + * labeled alternative in {@link SqlBaseParser#constant}. + * @param ctx the parse tree + */ + void exitGuidEscapedLiteral(SqlBaseParser.GuidEscapedLiteralContext ctx); /** * Enter a parse tree produced by {@link SqlBaseParser#comparisonOperator}. * @param ctx the parse tree diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseParser.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseParser.java index 3535977943ba7..32a1b062feebf 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseParser.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseParser.java @@ -1,15 +1,13 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ // ANTLR GENERATED CODE: DO NOT EDIT package org.elasticsearch.xpack.sql.parser; import org.antlr.v4.runtime.atn.*; import org.antlr.v4.runtime.dfa.DFA; import org.antlr.v4.runtime.*; +import org.antlr.v4.runtime.misc.*; import org.antlr.v4.runtime.tree.*; import java.util.List; +import java.util.Iterator; +import java.util.ArrayList; @SuppressWarnings({"all", "warnings", "unchecked", "unused", "cast"}) class SqlBaseParser extends Parser { @@ -28,33 +26,39 @@ class SqlBaseParser extends Parser { NOT=45, NULL=46, ON=47, OPTIMIZED=48, OR=49, ORDER=50, OUTER=51, PARSED=52, PHYSICAL=53, PLAN=54, RIGHT=55, RLIKE=56, QUERY=57, SCHEMAS=58, SELECT=59, SHOW=60, SYS=61, TABLE=62, TABLES=63, TEXT=64, TRUE=65, TYPE=66, TYPES=67, - USING=68, VERIFY=69, WHERE=70, WITH=71, EQ=72, NEQ=73, LT=74, LTE=75, - GT=76, GTE=77, PLUS=78, MINUS=79, ASTERISK=80, SLASH=81, PERCENT=82, CONCAT=83, - DOT=84, PARAM=85, STRING=86, INTEGER_VALUE=87, DECIMAL_VALUE=88, IDENTIFIER=89, - DIGIT_IDENTIFIER=90, TABLE_IDENTIFIER=91, QUOTED_IDENTIFIER=92, BACKQUOTED_IDENTIFIER=93, - SIMPLE_COMMENT=94, BRACKETED_COMMENT=95, WS=96, UNRECOGNIZED=97, DELIMITER=98; + USING=68, VERIFY=69, WHERE=70, WITH=71, ESCAPE_ESC=72, FUNCTION_ESC=73, + LIMIT_ESC=74, DATE_ESC=75, TIME_ESC=76, TIMESTAMP_ESC=77, GUID_ESC=78, + ESC_END=79, EQ=80, NEQ=81, LT=82, LTE=83, GT=84, GTE=85, PLUS=86, MINUS=87, + ASTERISK=88, SLASH=89, PERCENT=90, CONCAT=91, DOT=92, PARAM=93, STRING=94, + INTEGER_VALUE=95, DECIMAL_VALUE=96, IDENTIFIER=97, DIGIT_IDENTIFIER=98, + TABLE_IDENTIFIER=99, QUOTED_IDENTIFIER=100, BACKQUOTED_IDENTIFIER=101, + SIMPLE_COMMENT=102, BRACKETED_COMMENT=103, WS=104, UNRECOGNIZED=105, DELIMITER=106; public static final int RULE_singleStatement = 0, RULE_singleExpression = 1, RULE_statement = 2, - RULE_query = 3, RULE_queryNoWith = 4, RULE_queryTerm = 5, RULE_orderBy = 6, - RULE_querySpecification = 7, RULE_fromClause = 8, RULE_groupBy = 9, RULE_groupingElement = 10, - RULE_groupingExpressions = 11, RULE_namedQuery = 12, RULE_setQuantifier = 13, - RULE_selectItem = 14, RULE_relation = 15, RULE_joinRelation = 16, RULE_joinType = 17, - RULE_joinCriteria = 18, RULE_relationPrimary = 19, RULE_expression = 20, - RULE_booleanExpression = 21, RULE_predicated = 22, RULE_predicate = 23, - RULE_pattern = 24, RULE_valueExpression = 25, RULE_primaryExpression = 26, - RULE_constant = 27, RULE_comparisonOperator = 28, RULE_booleanValue = 29, - RULE_dataType = 30, RULE_qualifiedName = 31, RULE_identifier = 32, RULE_tableIdentifier = 33, - RULE_quoteIdentifier = 34, RULE_unquoteIdentifier = 35, RULE_number = 36, - RULE_string = 37, RULE_nonReserved = 38; + RULE_query = 3, RULE_queryNoWith = 4, RULE_limitClause = 5, RULE_queryTerm = 6, + RULE_orderBy = 7, RULE_querySpecification = 8, RULE_fromClause = 9, RULE_groupBy = 10, + RULE_groupingElement = 11, RULE_groupingExpressions = 12, RULE_namedQuery = 13, + RULE_setQuantifier = 14, RULE_selectItem = 15, RULE_relation = 16, RULE_joinRelation = 17, + RULE_joinType = 18, RULE_joinCriteria = 19, RULE_relationPrimary = 20, + RULE_expression = 21, RULE_booleanExpression = 22, RULE_predicated = 23, + RULE_predicate = 24, RULE_pattern = 25, RULE_patternEscape = 26, RULE_valueExpression = 27, + RULE_primaryExpression = 28, RULE_castExpression = 29, RULE_castTemplate = 30, + RULE_extractExpression = 31, RULE_extractTemplate = 32, RULE_functionExpression = 33, + RULE_functionTemplate = 34, RULE_constant = 35, RULE_comparisonOperator = 36, + RULE_booleanValue = 37, RULE_dataType = 38, RULE_qualifiedName = 39, RULE_identifier = 40, + RULE_tableIdentifier = 41, RULE_quoteIdentifier = 42, RULE_unquoteIdentifier = 43, + RULE_number = 44, RULE_string = 45, RULE_nonReserved = 46; public static final String[] ruleNames = { "singleStatement", "singleExpression", "statement", "query", "queryNoWith", - "queryTerm", "orderBy", "querySpecification", "fromClause", "groupBy", - "groupingElement", "groupingExpressions", "namedQuery", "setQuantifier", + "limitClause", "queryTerm", "orderBy", "querySpecification", "fromClause", + "groupBy", "groupingElement", "groupingExpressions", "namedQuery", "setQuantifier", "selectItem", "relation", "joinRelation", "joinType", "joinCriteria", "relationPrimary", "expression", "booleanExpression", "predicated", "predicate", - "pattern", "valueExpression", "primaryExpression", "constant", "comparisonOperator", - "booleanValue", "dataType", "qualifiedName", "identifier", "tableIdentifier", - "quoteIdentifier", "unquoteIdentifier", "number", "string", "nonReserved" + "pattern", "patternEscape", "valueExpression", "primaryExpression", "castExpression", + "castTemplate", "extractExpression", "extractTemplate", "functionExpression", + "functionTemplate", "constant", "comparisonOperator", "booleanValue", + "dataType", "qualifiedName", "identifier", "tableIdentifier", "quoteIdentifier", + "unquoteIdentifier", "number", "string", "nonReserved" }; private static final String[] _LITERAL_NAMES = { @@ -68,8 +72,9 @@ class SqlBaseParser extends Parser { "'OR'", "'ORDER'", "'OUTER'", "'PARSED'", "'PHYSICAL'", "'PLAN'", "'RIGHT'", "'RLIKE'", "'QUERY'", "'SCHEMAS'", "'SELECT'", "'SHOW'", "'SYS'", "'TABLE'", "'TABLES'", "'TEXT'", "'TRUE'", "'TYPE'", "'TYPES'", "'USING'", "'VERIFY'", - "'WHERE'", "'WITH'", "'='", null, "'<'", "'<='", "'>'", "'>='", "'+'", - "'-'", "'*'", "'/'", "'%'", "'||'", "'.'", "'?'" + "'WHERE'", "'WITH'", "'{ESCAPE'", "'{FN'", "'{LIMIT'", "'{D'", "'{T'", + "'{TS'", "'{GUID'", "'}'", "'='", null, "'<'", "'<='", "'>'", "'>='", + "'+'", "'-'", "'*'", "'/'", "'%'", "'||'", "'.'", "'?'" }; private static final String[] _SYMBOLIC_NAMES = { null, null, null, null, null, "ALL", "ANALYZE", "ANALYZED", "AND", "ANY", @@ -80,12 +85,13 @@ class SqlBaseParser extends Parser { "LIMIT", "MAPPED", "MATCH", "NATURAL", "NOT", "NULL", "ON", "OPTIMIZED", "OR", "ORDER", "OUTER", "PARSED", "PHYSICAL", "PLAN", "RIGHT", "RLIKE", "QUERY", "SCHEMAS", "SELECT", "SHOW", "SYS", "TABLE", "TABLES", "TEXT", - "TRUE", "TYPE", "TYPES", "USING", "VERIFY", "WHERE", "WITH", "EQ", "NEQ", - "LT", "LTE", "GT", "GTE", "PLUS", "MINUS", "ASTERISK", "SLASH", "PERCENT", - "CONCAT", "DOT", "PARAM", "STRING", "INTEGER_VALUE", "DECIMAL_VALUE", - "IDENTIFIER", "DIGIT_IDENTIFIER", "TABLE_IDENTIFIER", "QUOTED_IDENTIFIER", - "BACKQUOTED_IDENTIFIER", "SIMPLE_COMMENT", "BRACKETED_COMMENT", "WS", - "UNRECOGNIZED", "DELIMITER" + "TRUE", "TYPE", "TYPES", "USING", "VERIFY", "WHERE", "WITH", "ESCAPE_ESC", + "FUNCTION_ESC", "LIMIT_ESC", "DATE_ESC", "TIME_ESC", "TIMESTAMP_ESC", + "GUID_ESC", "ESC_END", "EQ", "NEQ", "LT", "LTE", "GT", "GTE", "PLUS", + "MINUS", "ASTERISK", "SLASH", "PERCENT", "CONCAT", "DOT", "PARAM", "STRING", + "INTEGER_VALUE", "DECIMAL_VALUE", "IDENTIFIER", "DIGIT_IDENTIFIER", "TABLE_IDENTIFIER", + "QUOTED_IDENTIFIER", "BACKQUOTED_IDENTIFIER", "SIMPLE_COMMENT", "BRACKETED_COMMENT", + "WS", "UNRECOGNIZED", "DELIMITER" }; public static final Vocabulary VOCABULARY = new VocabularyImpl(_LITERAL_NAMES, _SYMBOLIC_NAMES); @@ -166,9 +172,9 @@ public final SingleStatementContext singleStatement() throws RecognitionExceptio try { enterOuterAlt(_localctx, 1); { - setState(78); + setState(94); statement(); - setState(79); + setState(95); match(EOF); } } @@ -213,9 +219,9 @@ public final SingleExpressionContext singleExpression() throws RecognitionExcept try { enterOuterAlt(_localctx, 1); { - setState(81); + setState(97); expression(); - setState(82); + setState(98); match(EOF); } } @@ -601,14 +607,14 @@ public final StatementContext statement() throws RecognitionException { enterRule(_localctx, 4, RULE_statement); int _la; try { - setState(193); + setState(209); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,21,_ctx) ) { case 1: _localctx = new StatementDefaultContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(84); + setState(100); query(); } break; @@ -616,27 +622,27 @@ public final StatementContext statement() throws RecognitionException { _localctx = new ExplainContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(85); + setState(101); match(EXPLAIN); - setState(99); + setState(115); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,2,_ctx) ) { case 1: { - setState(86); + setState(102); match(T__0); - setState(95); + setState(111); _errHandler.sync(this); _la = _input.LA(1); while (((((_la - 28)) & ~0x3f) == 0 && ((1L << (_la - 28)) & ((1L << (FORMAT - 28)) | (1L << (PLAN - 28)) | (1L << (VERIFY - 28)))) != 0)) { { - setState(93); + setState(109); switch (_input.LA(1)) { case PLAN: { - setState(87); + setState(103); match(PLAN); - setState(88); + setState(104); ((ExplainContext)_localctx).type = _input.LT(1); _la = _input.LA(1); if ( !((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << ALL) | (1L << ANALYZED) | (1L << EXECUTABLE) | (1L << MAPPED) | (1L << OPTIMIZED) | (1L << PARSED))) != 0)) ) { @@ -648,9 +654,9 @@ public final StatementContext statement() throws RecognitionException { break; case FORMAT: { - setState(89); + setState(105); match(FORMAT); - setState(90); + setState(106); ((ExplainContext)_localctx).format = _input.LT(1); _la = _input.LA(1); if ( !(_la==GRAPHVIZ || _la==TEXT) ) { @@ -662,9 +668,9 @@ public final StatementContext statement() throws RecognitionException { break; case VERIFY: { - setState(91); + setState(107); match(VERIFY); - setState(92); + setState(108); ((ExplainContext)_localctx).verify = booleanValue(); } break; @@ -672,16 +678,16 @@ public final StatementContext statement() throws RecognitionException { throw new NoViableAltException(this); } } - setState(97); + setState(113); _errHandler.sync(this); _la = _input.LA(1); } - setState(98); + setState(114); match(T__1); } break; } - setState(101); + setState(117); statement(); } break; @@ -689,27 +695,27 @@ public final StatementContext statement() throws RecognitionException { _localctx = new DebugContext(_localctx); enterOuterAlt(_localctx, 3); { - setState(102); + setState(118); match(DEBUG); - setState(114); + setState(130); _errHandler.sync(this); switch ( getInterpreter().adaptivePredict(_input,5,_ctx) ) { case 1: { - setState(103); + setState(119); match(T__0); - setState(110); + setState(126); _errHandler.sync(this); _la = _input.LA(1); while (_la==FORMAT || _la==PLAN) { { - setState(108); + setState(124); switch (_input.LA(1)) { case PLAN: { - setState(104); + setState(120); match(PLAN); - setState(105); + setState(121); ((DebugContext)_localctx).type = _input.LT(1); _la = _input.LA(1); if ( !(_la==ANALYZED || _la==OPTIMIZED) ) { @@ -721,9 +727,9 @@ public final StatementContext statement() throws RecognitionException { break; case FORMAT: { - setState(106); + setState(122); match(FORMAT); - setState(107); + setState(123); ((DebugContext)_localctx).format = _input.LT(1); _la = _input.LA(1); if ( !(_la==GRAPHVIZ || _la==TEXT) ) { @@ -737,16 +743,16 @@ public final StatementContext statement() throws RecognitionException { throw new NoViableAltException(this); } } - setState(112); + setState(128); _errHandler.sync(this); _la = _input.LA(1); } - setState(113); + setState(129); match(T__1); } break; } - setState(116); + setState(132); statement(); } break; @@ -754,24 +760,24 @@ public final StatementContext statement() throws RecognitionException { _localctx = new ShowTablesContext(_localctx); enterOuterAlt(_localctx, 4); { - setState(117); + setState(133); match(SHOW); - setState(118); + setState(134); match(TABLES); - setState(123); + setState(139); _la = _input.LA(1); if (((((_la - 40)) & ~0x3f) == 0 && ((1L << (_la - 40)) & ((1L << (LIKE - 40)) | (1L << (PARAM - 40)) | (1L << (STRING - 40)))) != 0)) { { - setState(120); + setState(136); _la = _input.LA(1); if (_la==LIKE) { { - setState(119); + setState(135); match(LIKE); } } - setState(122); + setState(138); pattern(); } } @@ -782,18 +788,18 @@ public final StatementContext statement() throws RecognitionException { _localctx = new ShowColumnsContext(_localctx); enterOuterAlt(_localctx, 5); { - setState(125); + setState(141); match(SHOW); - setState(126); + setState(142); match(COLUMNS); - setState(127); + setState(143); _la = _input.LA(1); if ( !(_la==FROM || _la==IN) ) { _errHandler.recoverInline(this); } else { consume(); } - setState(128); + setState(144); tableIdentifier(); } break; @@ -801,14 +807,14 @@ public final StatementContext statement() throws RecognitionException { _localctx = new ShowColumnsContext(_localctx); enterOuterAlt(_localctx, 6); { - setState(129); + setState(145); _la = _input.LA(1); if ( !(_la==DESC || _la==DESCRIBE) ) { _errHandler.recoverInline(this); } else { consume(); } - setState(130); + setState(146); tableIdentifier(); } break; @@ -816,24 +822,24 @@ public final StatementContext statement() throws RecognitionException { _localctx = new ShowFunctionsContext(_localctx); enterOuterAlt(_localctx, 7); { - setState(131); + setState(147); match(SHOW); - setState(132); + setState(148); match(FUNCTIONS); - setState(137); + setState(153); _la = _input.LA(1); if (((((_la - 40)) & ~0x3f) == 0 && ((1L << (_la - 40)) & ((1L << (LIKE - 40)) | (1L << (PARAM - 40)) | (1L << (STRING - 40)))) != 0)) { { - setState(134); + setState(150); _la = _input.LA(1); if (_la==LIKE) { { - setState(133); + setState(149); match(LIKE); } } - setState(136); + setState(152); pattern(); } } @@ -844,9 +850,9 @@ public final StatementContext statement() throws RecognitionException { _localctx = new ShowSchemasContext(_localctx); enterOuterAlt(_localctx, 8); { - setState(139); + setState(155); match(SHOW); - setState(140); + setState(156); match(SCHEMAS); } break; @@ -854,9 +860,9 @@ public final StatementContext statement() throws RecognitionException { _localctx = new SysCatalogsContext(_localctx); enterOuterAlt(_localctx, 9); { - setState(141); + setState(157); match(SYS); - setState(142); + setState(158); match(CATALOGS); } break; @@ -864,69 +870,69 @@ public final StatementContext statement() throws RecognitionException { _localctx = new SysTablesContext(_localctx); enterOuterAlt(_localctx, 10); { - setState(143); + setState(159); match(SYS); - setState(144); + setState(160); match(TABLES); - setState(150); + setState(166); _la = _input.LA(1); if (_la==CATALOG) { { - setState(145); + setState(161); match(CATALOG); - setState(147); + setState(163); _la = _input.LA(1); if (_la==LIKE) { { - setState(146); + setState(162); match(LIKE); } } - setState(149); + setState(165); ((SysTablesContext)_localctx).clusterPattern = pattern(); } } - setState(156); + setState(172); _la = _input.LA(1); if (((((_la - 40)) & ~0x3f) == 0 && ((1L << (_la - 40)) & ((1L << (LIKE - 40)) | (1L << (PARAM - 40)) | (1L << (STRING - 40)))) != 0)) { { - setState(153); + setState(169); _la = _input.LA(1); if (_la==LIKE) { { - setState(152); + setState(168); match(LIKE); } } - setState(155); + setState(171); ((SysTablesContext)_localctx).tablePattern = pattern(); } } - setState(167); + setState(183); _la = _input.LA(1); if (_la==TYPE) { { - setState(158); + setState(174); match(TYPE); - setState(159); + setState(175); string(); - setState(164); + setState(180); _errHandler.sync(this); _la = _input.LA(1); while (_la==T__2) { { { - setState(160); + setState(176); match(T__2); - setState(161); + setState(177); string(); } } - setState(166); + setState(182); _errHandler.sync(this); _la = _input.LA(1); } @@ -939,55 +945,55 @@ public final StatementContext statement() throws RecognitionException { _localctx = new SysColumnsContext(_localctx); enterOuterAlt(_localctx, 11); { - setState(169); + setState(185); match(SYS); - setState(170); + setState(186); match(COLUMNS); - setState(173); + setState(189); _la = _input.LA(1); if (_la==CATALOG) { { - setState(171); + setState(187); match(CATALOG); - setState(172); + setState(188); ((SysColumnsContext)_localctx).cluster = string(); } } - setState(180); + setState(196); _la = _input.LA(1); if (_la==TABLE) { { - setState(175); + setState(191); match(TABLE); - setState(177); + setState(193); _la = _input.LA(1); if (_la==LIKE) { { - setState(176); + setState(192); match(LIKE); } } - setState(179); + setState(195); ((SysColumnsContext)_localctx).indexPattern = pattern(); } } - setState(186); + setState(202); _la = _input.LA(1); if (((((_la - 40)) & ~0x3f) == 0 && ((1L << (_la - 40)) & ((1L << (LIKE - 40)) | (1L << (PARAM - 40)) | (1L << (STRING - 40)))) != 0)) { { - setState(183); + setState(199); _la = _input.LA(1); if (_la==LIKE) { { - setState(182); + setState(198); match(LIKE); } } - setState(185); + setState(201); ((SysColumnsContext)_localctx).columnPattern = pattern(); } } @@ -998,9 +1004,9 @@ public final StatementContext statement() throws RecognitionException { _localctx = new SysTypesContext(_localctx); enterOuterAlt(_localctx, 12); { - setState(188); + setState(204); match(SYS); - setState(189); + setState(205); match(TYPES); } break; @@ -1008,11 +1014,11 @@ public final StatementContext statement() throws RecognitionException { _localctx = new SysTableTypesContext(_localctx); enterOuterAlt(_localctx, 13); { - setState(190); + setState(206); match(SYS); - setState(191); + setState(207); match(TABLE); - setState(192); + setState(208); match(TYPES); } break; @@ -1066,34 +1072,34 @@ public final QueryContext query() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(204); + setState(220); _la = _input.LA(1); if (_la==WITH) { { - setState(195); + setState(211); match(WITH); - setState(196); + setState(212); namedQuery(); - setState(201); + setState(217); _errHandler.sync(this); _la = _input.LA(1); while (_la==T__2) { { { - setState(197); + setState(213); match(T__2); - setState(198); + setState(214); namedQuery(); } } - setState(203); + setState(219); _errHandler.sync(this); _la = _input.LA(1); } } } - setState(206); + setState(222); queryNoWith(); } } @@ -1109,7 +1115,6 @@ public final QueryContext query() throws RecognitionException { } public static class QueryNoWithContext extends ParserRuleContext { - public Token limit; public QueryTermContext queryTerm() { return getRuleContext(QueryTermContext.class,0); } @@ -1121,9 +1126,9 @@ public List orderBy() { public OrderByContext orderBy(int i) { return getRuleContext(OrderByContext.class,i); } - public TerminalNode LIMIT() { return getToken(SqlBaseParser.LIMIT, 0); } - public TerminalNode INTEGER_VALUE() { return getToken(SqlBaseParser.INTEGER_VALUE, 0); } - public TerminalNode ALL() { return getToken(SqlBaseParser.ALL, 0); } + public LimitClauseContext limitClause() { + return getRuleContext(LimitClauseContext.class,0); + } public QueryNoWithContext(ParserRuleContext parent, int invokingState) { super(parent, invokingState); } @@ -1150,54 +1155,126 @@ public final QueryNoWithContext queryNoWith() throws RecognitionException { try { enterOuterAlt(_localctx, 1); { - setState(208); + setState(224); queryTerm(); - setState(219); + setState(235); _la = _input.LA(1); if (_la==ORDER) { { - setState(209); + setState(225); match(ORDER); - setState(210); + setState(226); match(BY); - setState(211); + setState(227); orderBy(); - setState(216); + setState(232); _errHandler.sync(this); _la = _input.LA(1); while (_la==T__2) { { { - setState(212); + setState(228); match(T__2); - setState(213); + setState(229); orderBy(); } } - setState(218); + setState(234); _errHandler.sync(this); _la = _input.LA(1); } } } - setState(223); + setState(238); _la = _input.LA(1); - if (_la==LIMIT) { + if (_la==LIMIT || _la==LIMIT_ESC) { + { + setState(237); + limitClause(); + } + } + + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class LimitClauseContext extends ParserRuleContext { + public Token limit; + public TerminalNode LIMIT() { return getToken(SqlBaseParser.LIMIT, 0); } + public TerminalNode INTEGER_VALUE() { return getToken(SqlBaseParser.INTEGER_VALUE, 0); } + public TerminalNode ALL() { return getToken(SqlBaseParser.ALL, 0); } + public TerminalNode LIMIT_ESC() { return getToken(SqlBaseParser.LIMIT_ESC, 0); } + public TerminalNode ESC_END() { return getToken(SqlBaseParser.ESC_END, 0); } + public LimitClauseContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_limitClause; } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof SqlBaseListener ) ((SqlBaseListener)listener).enterLimitClause(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof SqlBaseListener ) ((SqlBaseListener)listener).exitLimitClause(this); + } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof SqlBaseVisitor ) return ((SqlBaseVisitor)visitor).visitLimitClause(this); + else return visitor.visitChildren(this); + } + } + + public final LimitClauseContext limitClause() throws RecognitionException { + LimitClauseContext _localctx = new LimitClauseContext(_ctx, getState()); + enterRule(_localctx, 10, RULE_limitClause); + int _la; + try { + setState(245); + switch (_input.LA(1)) { + case LIMIT: + enterOuterAlt(_localctx, 1); { - setState(221); + setState(240); match(LIMIT); - setState(222); - ((QueryNoWithContext)_localctx).limit = _input.LT(1); + setState(241); + ((LimitClauseContext)_localctx).limit = _input.LT(1); _la = _input.LA(1); if ( !(_la==ALL || _la==INTEGER_VALUE) ) { - ((QueryNoWithContext)_localctx).limit = (Token)_errHandler.recoverInline(this); + ((LimitClauseContext)_localctx).limit = (Token)_errHandler.recoverInline(this); } else { consume(); } } - } - + break; + case LIMIT_ESC: + enterOuterAlt(_localctx, 2); + { + setState(242); + match(LIMIT_ESC); + setState(243); + ((LimitClauseContext)_localctx).limit = _input.LT(1); + _la = _input.LA(1); + if ( !(_la==ALL || _la==INTEGER_VALUE) ) { + ((LimitClauseContext)_localctx).limit = (Token)_errHandler.recoverInline(this); + } else { + consume(); + } + setState(244); + match(ESC_END); + } + break; + default: + throw new NoViableAltException(this); } } catch (RecognitionException re) { @@ -1263,15 +1340,15 @@ public T accept(ParseTreeVisitor visitor) { public final QueryTermContext queryTerm() throws RecognitionException { QueryTermContext _localctx = new QueryTermContext(_ctx, getState()); - enterRule(_localctx, 10, RULE_queryTerm); + enterRule(_localctx, 12, RULE_queryTerm); try { - setState(230); + setState(252); switch (_input.LA(1)) { case SELECT: _localctx = new QueryPrimaryDefaultContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(225); + setState(247); querySpecification(); } break; @@ -1279,11 +1356,11 @@ public final QueryTermContext queryTerm() throws RecognitionException { _localctx = new SubqueryContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(226); + setState(248); match(T__0); - setState(227); + setState(249); queryNoWith(); - setState(228); + setState(250); match(T__1); } break; @@ -1330,18 +1407,18 @@ public T accept(ParseTreeVisitor visitor) { public final OrderByContext orderBy() throws RecognitionException { OrderByContext _localctx = new OrderByContext(_ctx, getState()); - enterRule(_localctx, 12, RULE_orderBy); + enterRule(_localctx, 14, RULE_orderBy); int _la; try { enterOuterAlt(_localctx, 1); { - setState(232); + setState(254); expression(); - setState(234); + setState(256); _la = _input.LA(1); if (_la==ASC || _la==DESC) { { - setState(233); + setState(255); ((OrderByContext)_localctx).ordering = _input.LT(1); _la = _input.LA(1); if ( !(_la==ASC || _la==DESC) ) { @@ -1415,80 +1492,80 @@ public T accept(ParseTreeVisitor visitor) { public final QuerySpecificationContext querySpecification() throws RecognitionException { QuerySpecificationContext _localctx = new QuerySpecificationContext(_ctx, getState()); - enterRule(_localctx, 14, RULE_querySpecification); + enterRule(_localctx, 16, RULE_querySpecification); int _la; try { enterOuterAlt(_localctx, 1); { - setState(236); + setState(258); match(SELECT); - setState(238); + setState(260); _la = _input.LA(1); if (_la==ALL || _la==DISTINCT) { { - setState(237); + setState(259); setQuantifier(); } } - setState(240); + setState(262); selectItem(); - setState(245); + setState(267); _errHandler.sync(this); _la = _input.LA(1); while (_la==T__2) { { { - setState(241); + setState(263); match(T__2); - setState(242); + setState(264); selectItem(); } } - setState(247); + setState(269); _errHandler.sync(this); _la = _input.LA(1); } - setState(249); + setState(271); _la = _input.LA(1); if (_la==FROM) { { - setState(248); + setState(270); fromClause(); } } - setState(253); + setState(275); _la = _input.LA(1); if (_la==WHERE) { { - setState(251); + setState(273); match(WHERE); - setState(252); + setState(274); ((QuerySpecificationContext)_localctx).where = booleanExpression(0); } } - setState(258); + setState(280); _la = _input.LA(1); if (_la==GROUP) { { - setState(255); + setState(277); match(GROUP); - setState(256); + setState(278); match(BY); - setState(257); + setState(279); groupBy(); } } - setState(262); + setState(284); _la = _input.LA(1); if (_la==HAVING) { { - setState(260); + setState(282); match(HAVING); - setState(261); + setState(283); ((QuerySpecificationContext)_localctx).having = booleanExpression(0); } } @@ -1535,28 +1612,28 @@ public T accept(ParseTreeVisitor visitor) { public final FromClauseContext fromClause() throws RecognitionException { FromClauseContext _localctx = new FromClauseContext(_ctx, getState()); - enterRule(_localctx, 16, RULE_fromClause); + enterRule(_localctx, 18, RULE_fromClause); int _la; try { enterOuterAlt(_localctx, 1); { - setState(264); + setState(286); match(FROM); - setState(265); + setState(287); relation(); - setState(270); + setState(292); _errHandler.sync(this); _la = _input.LA(1); while (_la==T__2) { { { - setState(266); + setState(288); match(T__2); - setState(267); + setState(289); relation(); } } - setState(272); + setState(294); _errHandler.sync(this); _la = _input.LA(1); } @@ -1604,35 +1681,35 @@ public T accept(ParseTreeVisitor visitor) { public final GroupByContext groupBy() throws RecognitionException { GroupByContext _localctx = new GroupByContext(_ctx, getState()); - enterRule(_localctx, 18, RULE_groupBy); + enterRule(_localctx, 20, RULE_groupBy); int _la; try { enterOuterAlt(_localctx, 1); { - setState(274); + setState(296); _la = _input.LA(1); if (_la==ALL || _la==DISTINCT) { { - setState(273); + setState(295); setQuantifier(); } } - setState(276); + setState(298); groupingElement(); - setState(281); + setState(303); _errHandler.sync(this); _la = _input.LA(1); while (_la==T__2) { { { - setState(277); + setState(299); match(T__2); - setState(278); + setState(300); groupingElement(); } } - setState(283); + setState(305); _errHandler.sync(this); _la = _input.LA(1); } @@ -1682,12 +1759,12 @@ public T accept(ParseTreeVisitor visitor) { public final GroupingElementContext groupingElement() throws RecognitionException { GroupingElementContext _localctx = new GroupingElementContext(_ctx, getState()); - enterRule(_localctx, 20, RULE_groupingElement); + enterRule(_localctx, 22, RULE_groupingElement); try { _localctx = new SingleGroupingSetContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(284); + setState(306); groupingExpressions(); } } @@ -1730,50 +1807,50 @@ public T accept(ParseTreeVisitor visitor) { public final GroupingExpressionsContext groupingExpressions() throws RecognitionException { GroupingExpressionsContext _localctx = new GroupingExpressionsContext(_ctx, getState()); - enterRule(_localctx, 22, RULE_groupingExpressions); + enterRule(_localctx, 24, RULE_groupingExpressions); int _la; try { - setState(299); + setState(321); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,40,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,41,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(286); + setState(308); match(T__0); - setState(295); + setState(317); _la = _input.LA(1); - if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << T__0) | (1L << ANALYZE) | (1L << ANALYZED) | (1L << CAST) | (1L << CATALOGS) | (1L << COLUMNS) | (1L << DEBUG) | (1L << EXECUTABLE) | (1L << EXISTS) | (1L << EXPLAIN) | (1L << EXTRACT) | (1L << FALSE) | (1L << FORMAT) | (1L << FUNCTIONS) | (1L << GRAPHVIZ) | (1L << MAPPED) | (1L << MATCH) | (1L << NOT) | (1L << NULL) | (1L << OPTIMIZED) | (1L << PARSED) | (1L << PHYSICAL) | (1L << PLAN) | (1L << RLIKE) | (1L << QUERY) | (1L << SCHEMAS) | (1L << SHOW) | (1L << SYS) | (1L << TABLES))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (TEXT - 64)) | (1L << (TRUE - 64)) | (1L << (TYPE - 64)) | (1L << (TYPES - 64)) | (1L << (VERIFY - 64)) | (1L << (PLUS - 64)) | (1L << (MINUS - 64)) | (1L << (ASTERISK - 64)) | (1L << (PARAM - 64)) | (1L << (STRING - 64)) | (1L << (INTEGER_VALUE - 64)) | (1L << (DECIMAL_VALUE - 64)) | (1L << (IDENTIFIER - 64)) | (1L << (DIGIT_IDENTIFIER - 64)) | (1L << (QUOTED_IDENTIFIER - 64)) | (1L << (BACKQUOTED_IDENTIFIER - 64)))) != 0)) { + if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << T__0) | (1L << ANALYZE) | (1L << ANALYZED) | (1L << CAST) | (1L << CATALOGS) | (1L << COLUMNS) | (1L << DEBUG) | (1L << EXECUTABLE) | (1L << EXISTS) | (1L << EXPLAIN) | (1L << EXTRACT) | (1L << FALSE) | (1L << FORMAT) | (1L << FUNCTIONS) | (1L << GRAPHVIZ) | (1L << MAPPED) | (1L << MATCH) | (1L << NOT) | (1L << NULL) | (1L << OPTIMIZED) | (1L << PARSED) | (1L << PHYSICAL) | (1L << PLAN) | (1L << RLIKE) | (1L << QUERY) | (1L << SCHEMAS) | (1L << SHOW) | (1L << SYS) | (1L << TABLES))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (TEXT - 64)) | (1L << (TRUE - 64)) | (1L << (TYPE - 64)) | (1L << (TYPES - 64)) | (1L << (VERIFY - 64)) | (1L << (FUNCTION_ESC - 64)) | (1L << (DATE_ESC - 64)) | (1L << (TIME_ESC - 64)) | (1L << (TIMESTAMP_ESC - 64)) | (1L << (GUID_ESC - 64)) | (1L << (PLUS - 64)) | (1L << (MINUS - 64)) | (1L << (ASTERISK - 64)) | (1L << (PARAM - 64)) | (1L << (STRING - 64)) | (1L << (INTEGER_VALUE - 64)) | (1L << (DECIMAL_VALUE - 64)) | (1L << (IDENTIFIER - 64)) | (1L << (DIGIT_IDENTIFIER - 64)) | (1L << (QUOTED_IDENTIFIER - 64)) | (1L << (BACKQUOTED_IDENTIFIER - 64)))) != 0)) { { - setState(287); + setState(309); expression(); - setState(292); + setState(314); _errHandler.sync(this); _la = _input.LA(1); while (_la==T__2) { { { - setState(288); + setState(310); match(T__2); - setState(289); + setState(311); expression(); } } - setState(294); + setState(316); _errHandler.sync(this); _la = _input.LA(1); } } } - setState(297); + setState(319); match(T__1); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(298); + setState(320); expression(); } break; @@ -1820,19 +1897,19 @@ public T accept(ParseTreeVisitor visitor) { public final NamedQueryContext namedQuery() throws RecognitionException { NamedQueryContext _localctx = new NamedQueryContext(_ctx, getState()); - enterRule(_localctx, 24, RULE_namedQuery); + enterRule(_localctx, 26, RULE_namedQuery); try { enterOuterAlt(_localctx, 1); { - setState(301); + setState(323); ((NamedQueryContext)_localctx).name = identifier(); - setState(302); + setState(324); match(AS); - setState(303); + setState(325); match(T__0); - setState(304); + setState(326); queryNoWith(); - setState(305); + setState(327); match(T__1); } } @@ -1871,12 +1948,12 @@ public T accept(ParseTreeVisitor visitor) { public final SetQuantifierContext setQuantifier() throws RecognitionException { SetQuantifierContext _localctx = new SetQuantifierContext(_ctx, getState()); - enterRule(_localctx, 26, RULE_setQuantifier); + enterRule(_localctx, 28, RULE_setQuantifier); int _la; try { enterOuterAlt(_localctx, 1); { - setState(307); + setState(329); _la = _input.LA(1); if ( !(_la==ALL || _la==DISTINCT) ) { _errHandler.recoverInline(this); @@ -1933,28 +2010,28 @@ public T accept(ParseTreeVisitor visitor) { public final SelectItemContext selectItem() throws RecognitionException { SelectItemContext _localctx = new SelectItemContext(_ctx, getState()); - enterRule(_localctx, 28, RULE_selectItem); + enterRule(_localctx, 30, RULE_selectItem); int _la; try { _localctx = new SelectExpressionContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(309); + setState(331); expression(); - setState(314); + setState(336); _la = _input.LA(1); if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << ANALYZE) | (1L << ANALYZED) | (1L << AS) | (1L << CATALOGS) | (1L << COLUMNS) | (1L << DEBUG) | (1L << EXECUTABLE) | (1L << EXPLAIN) | (1L << FORMAT) | (1L << FUNCTIONS) | (1L << GRAPHVIZ) | (1L << MAPPED) | (1L << OPTIMIZED) | (1L << PARSED) | (1L << PHYSICAL) | (1L << PLAN) | (1L << RLIKE) | (1L << QUERY) | (1L << SCHEMAS) | (1L << SHOW) | (1L << SYS) | (1L << TABLES))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (TEXT - 64)) | (1L << (TYPE - 64)) | (1L << (TYPES - 64)) | (1L << (VERIFY - 64)) | (1L << (IDENTIFIER - 64)) | (1L << (DIGIT_IDENTIFIER - 64)) | (1L << (QUOTED_IDENTIFIER - 64)) | (1L << (BACKQUOTED_IDENTIFIER - 64)))) != 0)) { { - setState(311); + setState(333); _la = _input.LA(1); if (_la==AS) { { - setState(310); + setState(332); match(AS); } } - setState(313); + setState(335); identifier(); } } @@ -2003,24 +2080,24 @@ public T accept(ParseTreeVisitor visitor) { public final RelationContext relation() throws RecognitionException { RelationContext _localctx = new RelationContext(_ctx, getState()); - enterRule(_localctx, 30, RULE_relation); + enterRule(_localctx, 32, RULE_relation); int _la; try { enterOuterAlt(_localctx, 1); { - setState(316); + setState(338); relationPrimary(); - setState(320); + setState(342); _errHandler.sync(this); _la = _input.LA(1); while ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << FULL) | (1L << INNER) | (1L << JOIN) | (1L << LEFT) | (1L << NATURAL) | (1L << RIGHT))) != 0)) { { { - setState(317); + setState(339); joinRelation(); } } - setState(322); + setState(344); _errHandler.sync(this); _la = _input.LA(1); } @@ -2071,10 +2148,10 @@ public T accept(ParseTreeVisitor visitor) { public final JoinRelationContext joinRelation() throws RecognitionException { JoinRelationContext _localctx = new JoinRelationContext(_ctx, getState()); - enterRule(_localctx, 32, RULE_joinRelation); + enterRule(_localctx, 34, RULE_joinRelation); int _la; try { - setState(334); + setState(356); switch (_input.LA(1)) { case FULL: case INNER: @@ -2084,18 +2161,18 @@ public final JoinRelationContext joinRelation() throws RecognitionException { enterOuterAlt(_localctx, 1); { { - setState(323); + setState(345); joinType(); } - setState(324); + setState(346); match(JOIN); - setState(325); + setState(347); ((JoinRelationContext)_localctx).right = relationPrimary(); - setState(327); + setState(349); _la = _input.LA(1); if (_la==ON || _la==USING) { { - setState(326); + setState(348); joinCriteria(); } } @@ -2105,13 +2182,13 @@ public final JoinRelationContext joinRelation() throws RecognitionException { case NATURAL: enterOuterAlt(_localctx, 2); { - setState(329); + setState(351); match(NATURAL); - setState(330); + setState(352); joinType(); - setState(331); + setState(353); match(JOIN); - setState(332); + setState(354); ((JoinRelationContext)_localctx).right = relationPrimary(); } break; @@ -2157,20 +2234,20 @@ public T accept(ParseTreeVisitor visitor) { public final JoinTypeContext joinType() throws RecognitionException { JoinTypeContext _localctx = new JoinTypeContext(_ctx, getState()); - enterRule(_localctx, 34, RULE_joinType); + enterRule(_localctx, 36, RULE_joinType); int _la; try { - setState(351); + setState(373); switch (_input.LA(1)) { case INNER: case JOIN: enterOuterAlt(_localctx, 1); { - setState(337); + setState(359); _la = _input.LA(1); if (_la==INNER) { { - setState(336); + setState(358); match(INNER); } } @@ -2180,13 +2257,13 @@ public final JoinTypeContext joinType() throws RecognitionException { case LEFT: enterOuterAlt(_localctx, 2); { - setState(339); + setState(361); match(LEFT); - setState(341); + setState(363); _la = _input.LA(1); if (_la==OUTER) { { - setState(340); + setState(362); match(OUTER); } } @@ -2196,13 +2273,13 @@ public final JoinTypeContext joinType() throws RecognitionException { case RIGHT: enterOuterAlt(_localctx, 3); { - setState(343); + setState(365); match(RIGHT); - setState(345); + setState(367); _la = _input.LA(1); if (_la==OUTER) { { - setState(344); + setState(366); match(OUTER); } } @@ -2212,13 +2289,13 @@ public final JoinTypeContext joinType() throws RecognitionException { case FULL: enterOuterAlt(_localctx, 4); { - setState(347); + setState(369); match(FULL); - setState(349); + setState(371); _la = _input.LA(1); if (_la==OUTER) { { - setState(348); + setState(370); match(OUTER); } } @@ -2273,46 +2350,46 @@ public T accept(ParseTreeVisitor visitor) { public final JoinCriteriaContext joinCriteria() throws RecognitionException { JoinCriteriaContext _localctx = new JoinCriteriaContext(_ctx, getState()); - enterRule(_localctx, 36, RULE_joinCriteria); + enterRule(_localctx, 38, RULE_joinCriteria); int _la; try { - setState(367); + setState(389); switch (_input.LA(1)) { case ON: enterOuterAlt(_localctx, 1); { - setState(353); + setState(375); match(ON); - setState(354); + setState(376); booleanExpression(0); } break; case USING: enterOuterAlt(_localctx, 2); { - setState(355); + setState(377); match(USING); - setState(356); + setState(378); match(T__0); - setState(357); + setState(379); identifier(); - setState(362); + setState(384); _errHandler.sync(this); _la = _input.LA(1); while (_la==T__2) { { { - setState(358); + setState(380); match(T__2); - setState(359); + setState(381); identifier(); } } - setState(364); + setState(386); _errHandler.sync(this); _la = _input.LA(1); } - setState(365); + setState(387); match(T__1); } break; @@ -2414,32 +2491,32 @@ public T accept(ParseTreeVisitor visitor) { public final RelationPrimaryContext relationPrimary() throws RecognitionException { RelationPrimaryContext _localctx = new RelationPrimaryContext(_ctx, getState()); - enterRule(_localctx, 38, RULE_relationPrimary); + enterRule(_localctx, 40, RULE_relationPrimary); int _la; try { - setState(394); + setState(416); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,59,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,60,_ctx) ) { case 1: _localctx = new TableNameContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(369); + setState(391); tableIdentifier(); - setState(374); + setState(396); _la = _input.LA(1); if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << ANALYZE) | (1L << ANALYZED) | (1L << AS) | (1L << CATALOGS) | (1L << COLUMNS) | (1L << DEBUG) | (1L << EXECUTABLE) | (1L << EXPLAIN) | (1L << FORMAT) | (1L << FUNCTIONS) | (1L << GRAPHVIZ) | (1L << MAPPED) | (1L << OPTIMIZED) | (1L << PARSED) | (1L << PHYSICAL) | (1L << PLAN) | (1L << RLIKE) | (1L << QUERY) | (1L << SCHEMAS) | (1L << SHOW) | (1L << SYS) | (1L << TABLES))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (TEXT - 64)) | (1L << (TYPE - 64)) | (1L << (TYPES - 64)) | (1L << (VERIFY - 64)) | (1L << (IDENTIFIER - 64)) | (1L << (DIGIT_IDENTIFIER - 64)) | (1L << (QUOTED_IDENTIFIER - 64)) | (1L << (BACKQUOTED_IDENTIFIER - 64)))) != 0)) { { - setState(371); + setState(393); _la = _input.LA(1); if (_la==AS) { { - setState(370); + setState(392); match(AS); } } - setState(373); + setState(395); qualifiedName(); } } @@ -2450,26 +2527,26 @@ public final RelationPrimaryContext relationPrimary() throws RecognitionExceptio _localctx = new AliasedQueryContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(376); + setState(398); match(T__0); - setState(377); + setState(399); queryNoWith(); - setState(378); + setState(400); match(T__1); - setState(383); + setState(405); _la = _input.LA(1); if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << ANALYZE) | (1L << ANALYZED) | (1L << AS) | (1L << CATALOGS) | (1L << COLUMNS) | (1L << DEBUG) | (1L << EXECUTABLE) | (1L << EXPLAIN) | (1L << FORMAT) | (1L << FUNCTIONS) | (1L << GRAPHVIZ) | (1L << MAPPED) | (1L << OPTIMIZED) | (1L << PARSED) | (1L << PHYSICAL) | (1L << PLAN) | (1L << RLIKE) | (1L << QUERY) | (1L << SCHEMAS) | (1L << SHOW) | (1L << SYS) | (1L << TABLES))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (TEXT - 64)) | (1L << (TYPE - 64)) | (1L << (TYPES - 64)) | (1L << (VERIFY - 64)) | (1L << (IDENTIFIER - 64)) | (1L << (DIGIT_IDENTIFIER - 64)) | (1L << (QUOTED_IDENTIFIER - 64)) | (1L << (BACKQUOTED_IDENTIFIER - 64)))) != 0)) { { - setState(380); + setState(402); _la = _input.LA(1); if (_la==AS) { { - setState(379); + setState(401); match(AS); } } - setState(382); + setState(404); qualifiedName(); } } @@ -2480,26 +2557,26 @@ public final RelationPrimaryContext relationPrimary() throws RecognitionExceptio _localctx = new AliasedRelationContext(_localctx); enterOuterAlt(_localctx, 3); { - setState(385); + setState(407); match(T__0); - setState(386); + setState(408); relation(); - setState(387); + setState(409); match(T__1); - setState(392); + setState(414); _la = _input.LA(1); if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << ANALYZE) | (1L << ANALYZED) | (1L << AS) | (1L << CATALOGS) | (1L << COLUMNS) | (1L << DEBUG) | (1L << EXECUTABLE) | (1L << EXPLAIN) | (1L << FORMAT) | (1L << FUNCTIONS) | (1L << GRAPHVIZ) | (1L << MAPPED) | (1L << OPTIMIZED) | (1L << PARSED) | (1L << PHYSICAL) | (1L << PLAN) | (1L << RLIKE) | (1L << QUERY) | (1L << SCHEMAS) | (1L << SHOW) | (1L << SYS) | (1L << TABLES))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (TEXT - 64)) | (1L << (TYPE - 64)) | (1L << (TYPES - 64)) | (1L << (VERIFY - 64)) | (1L << (IDENTIFIER - 64)) | (1L << (DIGIT_IDENTIFIER - 64)) | (1L << (QUOTED_IDENTIFIER - 64)) | (1L << (BACKQUOTED_IDENTIFIER - 64)))) != 0)) { { - setState(389); + setState(411); _la = _input.LA(1); if (_la==AS) { { - setState(388); + setState(410); match(AS); } } - setState(391); + setState(413); qualifiedName(); } } @@ -2544,11 +2621,11 @@ public T accept(ParseTreeVisitor visitor) { public final ExpressionContext expression() throws RecognitionException { ExpressionContext _localctx = new ExpressionContext(_ctx, getState()); - enterRule(_localctx, 40, RULE_expression); + enterRule(_localctx, 42, RULE_expression); try { enterOuterAlt(_localctx, 1); { - setState(396); + setState(418); booleanExpression(0); } } @@ -2750,25 +2827,25 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc int _parentState = getState(); BooleanExpressionContext _localctx = new BooleanExpressionContext(_ctx, _parentState); BooleanExpressionContext _prevctx = _localctx; - int _startState = 42; - enterRecursionRule(_localctx, 42, RULE_booleanExpression, _p); + int _startState = 44; + enterRecursionRule(_localctx, 44, RULE_booleanExpression, _p); int _la; try { int _alt; enterOuterAlt(_localctx, 1); { - setState(447); + setState(469); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,63,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,64,_ctx) ) { case 1: { _localctx = new LogicalNotContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(399); + setState(421); match(NOT); - setState(400); + setState(422); booleanExpression(8); } break; @@ -2777,13 +2854,13 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new ExistsContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(401); + setState(423); match(EXISTS); - setState(402); + setState(424); match(T__0); - setState(403); + setState(425); query(); - setState(404); + setState(426); match(T__1); } break; @@ -2792,29 +2869,29 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new StringQueryContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(406); + setState(428); match(QUERY); - setState(407); + setState(429); match(T__0); - setState(408); + setState(430); ((StringQueryContext)_localctx).queryString = string(); - setState(413); + setState(435); _errHandler.sync(this); _la = _input.LA(1); while (_la==T__2) { { { - setState(409); + setState(431); match(T__2); - setState(410); + setState(432); ((StringQueryContext)_localctx).options = string(); } } - setState(415); + setState(437); _errHandler.sync(this); _la = _input.LA(1); } - setState(416); + setState(438); match(T__1); } break; @@ -2823,33 +2900,33 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new MatchQueryContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(418); + setState(440); match(MATCH); - setState(419); + setState(441); match(T__0); - setState(420); + setState(442); ((MatchQueryContext)_localctx).singleField = qualifiedName(); - setState(421); + setState(443); match(T__2); - setState(422); + setState(444); ((MatchQueryContext)_localctx).queryString = string(); - setState(427); + setState(449); _errHandler.sync(this); _la = _input.LA(1); while (_la==T__2) { { { - setState(423); + setState(445); match(T__2); - setState(424); + setState(446); ((MatchQueryContext)_localctx).options = string(); } } - setState(429); + setState(451); _errHandler.sync(this); _la = _input.LA(1); } - setState(430); + setState(452); match(T__1); } break; @@ -2858,33 +2935,33 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new MultiMatchQueryContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(432); + setState(454); match(MATCH); - setState(433); + setState(455); match(T__0); - setState(434); + setState(456); ((MultiMatchQueryContext)_localctx).multiFields = string(); - setState(435); + setState(457); match(T__2); - setState(436); + setState(458); ((MultiMatchQueryContext)_localctx).queryString = string(); - setState(441); + setState(463); _errHandler.sync(this); _la = _input.LA(1); while (_la==T__2) { { { - setState(437); + setState(459); match(T__2); - setState(438); + setState(460); ((MultiMatchQueryContext)_localctx).options = string(); } } - setState(443); + setState(465); _errHandler.sync(this); _la = _input.LA(1); } - setState(444); + setState(466); match(T__1); } break; @@ -2893,33 +2970,33 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new BooleanDefaultContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(446); + setState(468); predicated(); } break; } _ctx.stop = _input.LT(-1); - setState(457); + setState(479); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,65,_ctx); + _alt = getInterpreter().adaptivePredict(_input,66,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { if ( _parseListeners!=null ) triggerExitRuleEvent(); _prevctx = _localctx; { - setState(455); + setState(477); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,64,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,65,_ctx) ) { case 1: { _localctx = new LogicalBinaryContext(new BooleanExpressionContext(_parentctx, _parentState)); ((LogicalBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_booleanExpression); - setState(449); + setState(471); if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)"); - setState(450); + setState(472); ((LogicalBinaryContext)_localctx).operator = match(AND); - setState(451); + setState(473); ((LogicalBinaryContext)_localctx).right = booleanExpression(3); } break; @@ -2928,20 +3005,20 @@ private BooleanExpressionContext booleanExpression(int _p) throws RecognitionExc _localctx = new LogicalBinaryContext(new BooleanExpressionContext(_parentctx, _parentState)); ((LogicalBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_booleanExpression); - setState(452); + setState(474); if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)"); - setState(453); + setState(475); ((LogicalBinaryContext)_localctx).operator = match(OR); - setState(454); + setState(476); ((LogicalBinaryContext)_localctx).right = booleanExpression(2); } break; } } } - setState(459); + setState(481); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,65,_ctx); + _alt = getInterpreter().adaptivePredict(_input,66,_ctx); } } } @@ -2984,18 +3061,18 @@ public T accept(ParseTreeVisitor visitor) { public final PredicatedContext predicated() throws RecognitionException { PredicatedContext _localctx = new PredicatedContext(_ctx, getState()); - enterRule(_localctx, 44, RULE_predicated); + enterRule(_localctx, 46, RULE_predicated); try { enterOuterAlt(_localctx, 1); { - setState(460); + setState(482); valueExpression(0); - setState(462); + setState(484); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,66,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,67,_ctx) ) { case 1: { - setState(461); + setState(483); predicate(); } break; @@ -3068,145 +3145,145 @@ public T accept(ParseTreeVisitor visitor) { public final PredicateContext predicate() throws RecognitionException { PredicateContext _localctx = new PredicateContext(_ctx, getState()); - enterRule(_localctx, 46, RULE_predicate); + enterRule(_localctx, 48, RULE_predicate); int _la; try { - setState(510); + setState(532); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,74,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,75,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(465); + setState(487); _la = _input.LA(1); if (_la==NOT) { { - setState(464); + setState(486); match(NOT); } } - setState(467); + setState(489); ((PredicateContext)_localctx).kind = match(BETWEEN); - setState(468); + setState(490); ((PredicateContext)_localctx).lower = valueExpression(0); - setState(469); + setState(491); match(AND); - setState(470); + setState(492); ((PredicateContext)_localctx).upper = valueExpression(0); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(473); + setState(495); _la = _input.LA(1); if (_la==NOT) { { - setState(472); + setState(494); match(NOT); } } - setState(475); + setState(497); ((PredicateContext)_localctx).kind = match(IN); - setState(476); + setState(498); match(T__0); - setState(477); + setState(499); expression(); - setState(482); + setState(504); _errHandler.sync(this); _la = _input.LA(1); while (_la==T__2) { { { - setState(478); + setState(500); match(T__2); - setState(479); + setState(501); expression(); } } - setState(484); + setState(506); _errHandler.sync(this); _la = _input.LA(1); } - setState(485); + setState(507); match(T__1); } break; case 3: enterOuterAlt(_localctx, 3); { - setState(488); + setState(510); _la = _input.LA(1); if (_la==NOT) { { - setState(487); + setState(509); match(NOT); } } - setState(490); + setState(512); ((PredicateContext)_localctx).kind = match(IN); - setState(491); + setState(513); match(T__0); - setState(492); + setState(514); query(); - setState(493); + setState(515); match(T__1); } break; case 4: enterOuterAlt(_localctx, 4); { - setState(496); + setState(518); _la = _input.LA(1); if (_la==NOT) { { - setState(495); + setState(517); match(NOT); } } - setState(498); + setState(520); ((PredicateContext)_localctx).kind = match(LIKE); - setState(499); + setState(521); pattern(); } break; case 5: enterOuterAlt(_localctx, 5); { - setState(501); + setState(523); _la = _input.LA(1); if (_la==NOT) { { - setState(500); + setState(522); match(NOT); } } - setState(503); + setState(525); ((PredicateContext)_localctx).kind = match(RLIKE); - setState(504); + setState(526); ((PredicateContext)_localctx).regex = string(); } break; case 6: enterOuterAlt(_localctx, 6); { - setState(505); + setState(527); match(IS); - setState(507); + setState(529); _la = _input.LA(1); if (_la==NOT) { { - setState(506); + setState(528); match(NOT); } } - setState(509); + setState(531); ((PredicateContext)_localctx).kind = match(NULL); } break; @@ -3225,14 +3302,12 @@ public final PredicateContext predicate() throws RecognitionException { public static class PatternContext extends ParserRuleContext { public StringContext value; - public StringContext escape; - public List string() { - return getRuleContexts(StringContext.class); + public StringContext string() { + return getRuleContext(StringContext.class,0); } - public StringContext string(int i) { - return getRuleContext(StringContext.class,i); + public PatternEscapeContext patternEscape() { + return getRuleContext(PatternEscapeContext.class,0); } - public TerminalNode ESCAPE() { return getToken(SqlBaseParser.ESCAPE, 0); } public PatternContext(ParserRuleContext parent, int invokingState) { super(parent, invokingState); } @@ -3254,21 +3329,19 @@ public T accept(ParseTreeVisitor visitor) { public final PatternContext pattern() throws RecognitionException { PatternContext _localctx = new PatternContext(_ctx, getState()); - enterRule(_localctx, 48, RULE_pattern); + enterRule(_localctx, 50, RULE_pattern); try { enterOuterAlt(_localctx, 1); { - setState(512); + setState(534); ((PatternContext)_localctx).value = string(); - setState(515); + setState(536); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,75,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,76,_ctx) ) { case 1: { - setState(513); - match(ESCAPE); - setState(514); - ((PatternContext)_localctx).escape = string(); + setState(535); + patternEscape(); } break; } @@ -3285,19 +3358,86 @@ public final PatternContext pattern() throws RecognitionException { return _localctx; } - public static class ValueExpressionContext extends ParserRuleContext { - public ValueExpressionContext(ParserRuleContext parent, int invokingState) { + public static class PatternEscapeContext extends ParserRuleContext { + public StringContext escape; + public TerminalNode ESCAPE() { return getToken(SqlBaseParser.ESCAPE, 0); } + public StringContext string() { + return getRuleContext(StringContext.class,0); + } + public TerminalNode ESCAPE_ESC() { return getToken(SqlBaseParser.ESCAPE_ESC, 0); } + public PatternEscapeContext(ParserRuleContext parent, int invokingState) { super(parent, invokingState); } - @Override public int getRuleIndex() { return RULE_valueExpression; } - - public ValueExpressionContext() { } - public void copyFrom(ValueExpressionContext ctx) { - super.copyFrom(ctx); + @Override public int getRuleIndex() { return RULE_patternEscape; } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof SqlBaseListener ) ((SqlBaseListener)listener).enterPatternEscape(this); } - } - public static class ValueExpressionDefaultContext extends ValueExpressionContext { - public PrimaryExpressionContext primaryExpression() { + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof SqlBaseListener ) ((SqlBaseListener)listener).exitPatternEscape(this); + } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof SqlBaseVisitor ) return ((SqlBaseVisitor)visitor).visitPatternEscape(this); + else return visitor.visitChildren(this); + } + } + + public final PatternEscapeContext patternEscape() throws RecognitionException { + PatternEscapeContext _localctx = new PatternEscapeContext(_ctx, getState()); + enterRule(_localctx, 52, RULE_patternEscape); + try { + setState(544); + switch (_input.LA(1)) { + case ESCAPE: + enterOuterAlt(_localctx, 1); + { + setState(538); + match(ESCAPE); + setState(539); + ((PatternEscapeContext)_localctx).escape = string(); + } + break; + case ESCAPE_ESC: + enterOuterAlt(_localctx, 2); + { + setState(540); + match(ESCAPE_ESC); + setState(541); + ((PatternEscapeContext)_localctx).escape = string(); + setState(542); + match(ESC_END); + } + break; + default: + throw new NoViableAltException(this); + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class ValueExpressionContext extends ParserRuleContext { + public ValueExpressionContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_valueExpression; } + + public ValueExpressionContext() { } + public void copyFrom(ValueExpressionContext ctx) { + super.copyFrom(ctx); + } + } + public static class ValueExpressionDefaultContext extends ValueExpressionContext { + public PrimaryExpressionContext primaryExpression() { return getRuleContext(PrimaryExpressionContext.class,0); } public ValueExpressionDefaultContext(ValueExpressionContext ctx) { copyFrom(ctx); } @@ -3404,14 +3544,14 @@ private ValueExpressionContext valueExpression(int _p) throws RecognitionExcepti int _parentState = getState(); ValueExpressionContext _localctx = new ValueExpressionContext(_ctx, _parentState); ValueExpressionContext _prevctx = _localctx; - int _startState = 50; - enterRecursionRule(_localctx, 50, RULE_valueExpression, _p); + int _startState = 54; + enterRecursionRule(_localctx, 54, RULE_valueExpression, _p); int _la; try { int _alt; enterOuterAlt(_localctx, 1); { - setState(521); + setState(550); switch (_input.LA(1)) { case T__0: case ANALYZE: @@ -3444,6 +3584,11 @@ private ValueExpressionContext valueExpression(int _p) throws RecognitionExcepti case TYPE: case TYPES: case VERIFY: + case FUNCTION_ESC: + case DATE_ESC: + case TIME_ESC: + case TIMESTAMP_ESC: + case GUID_ESC: case ASTERISK: case PARAM: case STRING: @@ -3458,7 +3603,7 @@ private ValueExpressionContext valueExpression(int _p) throws RecognitionExcepti _ctx = _localctx; _prevctx = _localctx; - setState(518); + setState(547); primaryExpression(); } break; @@ -3468,7 +3613,7 @@ private ValueExpressionContext valueExpression(int _p) throws RecognitionExcepti _localctx = new ArithmeticUnaryContext(_localctx); _ctx = _localctx; _prevctx = _localctx; - setState(519); + setState(548); ((ArithmeticUnaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { @@ -3476,7 +3621,7 @@ private ValueExpressionContext valueExpression(int _p) throws RecognitionExcepti } else { consume(); } - setState(520); + setState(549); valueExpression(4); } break; @@ -3484,33 +3629,33 @@ private ValueExpressionContext valueExpression(int _p) throws RecognitionExcepti throw new NoViableAltException(this); } _ctx.stop = _input.LT(-1); - setState(535); + setState(564); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,78,_ctx); + _alt = getInterpreter().adaptivePredict(_input,80,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { if ( _parseListeners!=null ) triggerExitRuleEvent(); _prevctx = _localctx; { - setState(533); + setState(562); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,77,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,79,_ctx) ) { case 1: { _localctx = new ArithmeticBinaryContext(new ValueExpressionContext(_parentctx, _parentState)); ((ArithmeticBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_valueExpression); - setState(523); + setState(552); if (!(precpred(_ctx, 3))) throw new FailedPredicateException(this, "precpred(_ctx, 3)"); - setState(524); + setState(553); ((ArithmeticBinaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); - if ( !(((((_la - 80)) & ~0x3f) == 0 && ((1L << (_la - 80)) & ((1L << (ASTERISK - 80)) | (1L << (SLASH - 80)) | (1L << (PERCENT - 80)))) != 0)) ) { + if ( !(((((_la - 88)) & ~0x3f) == 0 && ((1L << (_la - 88)) & ((1L << (ASTERISK - 88)) | (1L << (SLASH - 88)) | (1L << (PERCENT - 88)))) != 0)) ) { ((ArithmeticBinaryContext)_localctx).operator = (Token)_errHandler.recoverInline(this); } else { consume(); } - setState(525); + setState(554); ((ArithmeticBinaryContext)_localctx).right = valueExpression(4); } break; @@ -3519,9 +3664,9 @@ private ValueExpressionContext valueExpression(int _p) throws RecognitionExcepti _localctx = new ArithmeticBinaryContext(new ValueExpressionContext(_parentctx, _parentState)); ((ArithmeticBinaryContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_valueExpression); - setState(526); + setState(555); if (!(precpred(_ctx, 2))) throw new FailedPredicateException(this, "precpred(_ctx, 2)"); - setState(527); + setState(556); ((ArithmeticBinaryContext)_localctx).operator = _input.LT(1); _la = _input.LA(1); if ( !(_la==PLUS || _la==MINUS) ) { @@ -3529,7 +3674,7 @@ private ValueExpressionContext valueExpression(int _p) throws RecognitionExcepti } else { consume(); } - setState(528); + setState(557); ((ArithmeticBinaryContext)_localctx).right = valueExpression(3); } break; @@ -3538,20 +3683,20 @@ private ValueExpressionContext valueExpression(int _p) throws RecognitionExcepti _localctx = new ComparisonContext(new ValueExpressionContext(_parentctx, _parentState)); ((ComparisonContext)_localctx).left = _prevctx; pushNewRecursionContext(_localctx, _startState, RULE_valueExpression); - setState(529); + setState(558); if (!(precpred(_ctx, 1))) throw new FailedPredicateException(this, "precpred(_ctx, 1)"); - setState(530); + setState(559); comparisonOperator(); - setState(531); + setState(560); ((ComparisonContext)_localctx).right = valueExpression(2); } break; } } } - setState(537); + setState(566); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,78,_ctx); + _alt = getInterpreter().adaptivePredict(_input,80,_ctx); } } } @@ -3597,13 +3742,8 @@ public T accept(ParseTreeVisitor visitor) { } } public static class CastContext extends PrimaryExpressionContext { - public TerminalNode CAST() { return getToken(SqlBaseParser.CAST, 0); } - public ExpressionContext expression() { - return getRuleContext(ExpressionContext.class,0); - } - public TerminalNode AS() { return getToken(SqlBaseParser.AS, 0); } - public DataTypeContext dataType() { - return getRuleContext(DataTypeContext.class,0); + public CastExpressionContext castExpression() { + return getRuleContext(CastExpressionContext.class,0); } public CastContext(PrimaryExpressionContext ctx) { copyFrom(ctx); } @Override @@ -3659,14 +3799,8 @@ public T accept(ParseTreeVisitor visitor) { } } public static class ExtractContext extends PrimaryExpressionContext { - public IdentifierContext field; - public TerminalNode EXTRACT() { return getToken(SqlBaseParser.EXTRACT, 0); } - public TerminalNode FROM() { return getToken(SqlBaseParser.FROM, 0); } - public ValueExpressionContext valueExpression() { - return getRuleContext(ValueExpressionContext.class,0); - } - public IdentifierContext identifier() { - return getRuleContext(IdentifierContext.class,0); + public ExtractExpressionContext extractExpression() { + return getRuleContext(ExtractExpressionContext.class,0); } public ExtractContext(PrimaryExpressionContext ctx) { copyFrom(ctx); } @Override @@ -3713,17 +3847,514 @@ public QualifiedNameContext qualifiedName() { public void enterRule(ParseTreeListener listener) { if ( listener instanceof SqlBaseListener ) ((SqlBaseListener)listener).enterStar(this); } - @Override - public void exitRule(ParseTreeListener listener) { - if ( listener instanceof SqlBaseListener ) ((SqlBaseListener)listener).exitStar(this); + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof SqlBaseListener ) ((SqlBaseListener)listener).exitStar(this); + } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof SqlBaseVisitor ) return ((SqlBaseVisitor)visitor).visitStar(this); + else return visitor.visitChildren(this); + } + } + public static class FunctionContext extends PrimaryExpressionContext { + public FunctionExpressionContext functionExpression() { + return getRuleContext(FunctionExpressionContext.class,0); + } + public FunctionContext(PrimaryExpressionContext ctx) { copyFrom(ctx); } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof SqlBaseListener ) ((SqlBaseListener)listener).enterFunction(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof SqlBaseListener ) ((SqlBaseListener)listener).exitFunction(this); + } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof SqlBaseVisitor ) return ((SqlBaseVisitor)visitor).visitFunction(this); + else return visitor.visitChildren(this); + } + } + public static class SubqueryExpressionContext extends PrimaryExpressionContext { + public QueryContext query() { + return getRuleContext(QueryContext.class,0); + } + public SubqueryExpressionContext(PrimaryExpressionContext ctx) { copyFrom(ctx); } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof SqlBaseListener ) ((SqlBaseListener)listener).enterSubqueryExpression(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof SqlBaseListener ) ((SqlBaseListener)listener).exitSubqueryExpression(this); + } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof SqlBaseVisitor ) return ((SqlBaseVisitor)visitor).visitSubqueryExpression(this); + else return visitor.visitChildren(this); + } + } + + public final PrimaryExpressionContext primaryExpression() throws RecognitionException { + PrimaryExpressionContext _localctx = new PrimaryExpressionContext(_ctx, getState()); + enterRule(_localctx, 56, RULE_primaryExpression); + int _la; + try { + setState(588); + _errHandler.sync(this); + switch ( getInterpreter().adaptivePredict(_input,82,_ctx) ) { + case 1: + _localctx = new CastContext(_localctx); + enterOuterAlt(_localctx, 1); + { + setState(567); + castExpression(); + } + break; + case 2: + _localctx = new ExtractContext(_localctx); + enterOuterAlt(_localctx, 2); + { + setState(568); + extractExpression(); + } + break; + case 3: + _localctx = new ConstantDefaultContext(_localctx); + enterOuterAlt(_localctx, 3); + { + setState(569); + constant(); + } + break; + case 4: + _localctx = new StarContext(_localctx); + enterOuterAlt(_localctx, 4); + { + setState(570); + match(ASTERISK); + } + break; + case 5: + _localctx = new StarContext(_localctx); + enterOuterAlt(_localctx, 5); + { + setState(574); + _la = _input.LA(1); + if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << ANALYZE) | (1L << ANALYZED) | (1L << CATALOGS) | (1L << COLUMNS) | (1L << DEBUG) | (1L << EXECUTABLE) | (1L << EXPLAIN) | (1L << FORMAT) | (1L << FUNCTIONS) | (1L << GRAPHVIZ) | (1L << MAPPED) | (1L << OPTIMIZED) | (1L << PARSED) | (1L << PHYSICAL) | (1L << PLAN) | (1L << RLIKE) | (1L << QUERY) | (1L << SCHEMAS) | (1L << SHOW) | (1L << SYS) | (1L << TABLES))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (TEXT - 64)) | (1L << (TYPE - 64)) | (1L << (TYPES - 64)) | (1L << (VERIFY - 64)) | (1L << (IDENTIFIER - 64)) | (1L << (DIGIT_IDENTIFIER - 64)) | (1L << (QUOTED_IDENTIFIER - 64)) | (1L << (BACKQUOTED_IDENTIFIER - 64)))) != 0)) { + { + setState(571); + qualifiedName(); + setState(572); + match(DOT); + } + } + + setState(576); + match(ASTERISK); + } + break; + case 6: + _localctx = new FunctionContext(_localctx); + enterOuterAlt(_localctx, 6); + { + setState(577); + functionExpression(); + } + break; + case 7: + _localctx = new SubqueryExpressionContext(_localctx); + enterOuterAlt(_localctx, 7); + { + setState(578); + match(T__0); + setState(579); + query(); + setState(580); + match(T__1); + } + break; + case 8: + _localctx = new ColumnReferenceContext(_localctx); + enterOuterAlt(_localctx, 8); + { + setState(582); + identifier(); + } + break; + case 9: + _localctx = new DereferenceContext(_localctx); + enterOuterAlt(_localctx, 9); + { + setState(583); + qualifiedName(); + } + break; + case 10: + _localctx = new ParenthesizedExpressionContext(_localctx); + enterOuterAlt(_localctx, 10); + { + setState(584); + match(T__0); + setState(585); + expression(); + setState(586); + match(T__1); + } + break; + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class CastExpressionContext extends ParserRuleContext { + public CastTemplateContext castTemplate() { + return getRuleContext(CastTemplateContext.class,0); + } + public TerminalNode FUNCTION_ESC() { return getToken(SqlBaseParser.FUNCTION_ESC, 0); } + public TerminalNode ESC_END() { return getToken(SqlBaseParser.ESC_END, 0); } + public CastExpressionContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_castExpression; } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof SqlBaseListener ) ((SqlBaseListener)listener).enterCastExpression(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof SqlBaseListener ) ((SqlBaseListener)listener).exitCastExpression(this); + } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof SqlBaseVisitor ) return ((SqlBaseVisitor)visitor).visitCastExpression(this); + else return visitor.visitChildren(this); + } + } + + public final CastExpressionContext castExpression() throws RecognitionException { + CastExpressionContext _localctx = new CastExpressionContext(_ctx, getState()); + enterRule(_localctx, 58, RULE_castExpression); + try { + setState(595); + switch (_input.LA(1)) { + case CAST: + enterOuterAlt(_localctx, 1); + { + setState(590); + castTemplate(); + } + break; + case FUNCTION_ESC: + enterOuterAlt(_localctx, 2); + { + setState(591); + match(FUNCTION_ESC); + setState(592); + castTemplate(); + setState(593); + match(ESC_END); + } + break; + default: + throw new NoViableAltException(this); + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class CastTemplateContext extends ParserRuleContext { + public TerminalNode CAST() { return getToken(SqlBaseParser.CAST, 0); } + public ExpressionContext expression() { + return getRuleContext(ExpressionContext.class,0); + } + public TerminalNode AS() { return getToken(SqlBaseParser.AS, 0); } + public DataTypeContext dataType() { + return getRuleContext(DataTypeContext.class,0); + } + public CastTemplateContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_castTemplate; } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof SqlBaseListener ) ((SqlBaseListener)listener).enterCastTemplate(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof SqlBaseListener ) ((SqlBaseListener)listener).exitCastTemplate(this); + } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof SqlBaseVisitor ) return ((SqlBaseVisitor)visitor).visitCastTemplate(this); + else return visitor.visitChildren(this); + } + } + + public final CastTemplateContext castTemplate() throws RecognitionException { + CastTemplateContext _localctx = new CastTemplateContext(_ctx, getState()); + enterRule(_localctx, 60, RULE_castTemplate); + try { + enterOuterAlt(_localctx, 1); + { + setState(597); + match(CAST); + setState(598); + match(T__0); + setState(599); + expression(); + setState(600); + match(AS); + setState(601); + dataType(); + setState(602); + match(T__1); + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class ExtractExpressionContext extends ParserRuleContext { + public ExtractTemplateContext extractTemplate() { + return getRuleContext(ExtractTemplateContext.class,0); + } + public TerminalNode FUNCTION_ESC() { return getToken(SqlBaseParser.FUNCTION_ESC, 0); } + public TerminalNode ESC_END() { return getToken(SqlBaseParser.ESC_END, 0); } + public ExtractExpressionContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_extractExpression; } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof SqlBaseListener ) ((SqlBaseListener)listener).enterExtractExpression(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof SqlBaseListener ) ((SqlBaseListener)listener).exitExtractExpression(this); + } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof SqlBaseVisitor ) return ((SqlBaseVisitor)visitor).visitExtractExpression(this); + else return visitor.visitChildren(this); + } + } + + public final ExtractExpressionContext extractExpression() throws RecognitionException { + ExtractExpressionContext _localctx = new ExtractExpressionContext(_ctx, getState()); + enterRule(_localctx, 62, RULE_extractExpression); + try { + setState(609); + switch (_input.LA(1)) { + case EXTRACT: + enterOuterAlt(_localctx, 1); + { + setState(604); + extractTemplate(); + } + break; + case FUNCTION_ESC: + enterOuterAlt(_localctx, 2); + { + setState(605); + match(FUNCTION_ESC); + setState(606); + extractTemplate(); + setState(607); + match(ESC_END); + } + break; + default: + throw new NoViableAltException(this); + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class ExtractTemplateContext extends ParserRuleContext { + public IdentifierContext field; + public TerminalNode EXTRACT() { return getToken(SqlBaseParser.EXTRACT, 0); } + public TerminalNode FROM() { return getToken(SqlBaseParser.FROM, 0); } + public ValueExpressionContext valueExpression() { + return getRuleContext(ValueExpressionContext.class,0); + } + public IdentifierContext identifier() { + return getRuleContext(IdentifierContext.class,0); + } + public ExtractTemplateContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_extractTemplate; } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof SqlBaseListener ) ((SqlBaseListener)listener).enterExtractTemplate(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof SqlBaseListener ) ((SqlBaseListener)listener).exitExtractTemplate(this); + } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof SqlBaseVisitor ) return ((SqlBaseVisitor)visitor).visitExtractTemplate(this); + else return visitor.visitChildren(this); + } + } + + public final ExtractTemplateContext extractTemplate() throws RecognitionException { + ExtractTemplateContext _localctx = new ExtractTemplateContext(_ctx, getState()); + enterRule(_localctx, 64, RULE_extractTemplate); + try { + enterOuterAlt(_localctx, 1); + { + setState(611); + match(EXTRACT); + setState(612); + match(T__0); + setState(613); + ((ExtractTemplateContext)_localctx).field = identifier(); + setState(614); + match(FROM); + setState(615); + valueExpression(0); + setState(616); + match(T__1); + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); + } + finally { + exitRule(); + } + return _localctx; + } + + public static class FunctionExpressionContext extends ParserRuleContext { + public FunctionTemplateContext functionTemplate() { + return getRuleContext(FunctionTemplateContext.class,0); + } + public TerminalNode FUNCTION_ESC() { return getToken(SqlBaseParser.FUNCTION_ESC, 0); } + public FunctionExpressionContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); + } + @Override public int getRuleIndex() { return RULE_functionExpression; } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof SqlBaseListener ) ((SqlBaseListener)listener).enterFunctionExpression(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof SqlBaseListener ) ((SqlBaseListener)listener).exitFunctionExpression(this); + } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof SqlBaseVisitor ) return ((SqlBaseVisitor)visitor).visitFunctionExpression(this); + else return visitor.visitChildren(this); + } + } + + public final FunctionExpressionContext functionExpression() throws RecognitionException { + FunctionExpressionContext _localctx = new FunctionExpressionContext(_ctx, getState()); + enterRule(_localctx, 66, RULE_functionExpression); + try { + setState(623); + switch (_input.LA(1)) { + case ANALYZE: + case ANALYZED: + case CATALOGS: + case COLUMNS: + case DEBUG: + case EXECUTABLE: + case EXPLAIN: + case FORMAT: + case FUNCTIONS: + case GRAPHVIZ: + case MAPPED: + case OPTIMIZED: + case PARSED: + case PHYSICAL: + case PLAN: + case RLIKE: + case QUERY: + case SCHEMAS: + case SHOW: + case SYS: + case TABLES: + case TEXT: + case TYPE: + case TYPES: + case VERIFY: + case IDENTIFIER: + case DIGIT_IDENTIFIER: + case QUOTED_IDENTIFIER: + case BACKQUOTED_IDENTIFIER: + enterOuterAlt(_localctx, 1); + { + setState(618); + functionTemplate(); + } + break; + case FUNCTION_ESC: + enterOuterAlt(_localctx, 2); + { + setState(619); + match(FUNCTION_ESC); + setState(620); + functionTemplate(); + setState(621); + match(ESC_END); + } + break; + default: + throw new NoViableAltException(this); + } + } + catch (RecognitionException re) { + _localctx.exception = re; + _errHandler.reportError(this, re); + _errHandler.recover(this, re); } - @Override - public T accept(ParseTreeVisitor visitor) { - if ( visitor instanceof SqlBaseVisitor ) return ((SqlBaseVisitor)visitor).visitStar(this); - else return visitor.visitChildren(this); + finally { + exitRule(); } + return _localctx; } - public static class FunctionCallContext extends PrimaryExpressionContext { + + public static class FunctionTemplateContext extends ParserRuleContext { public IdentifierContext identifier() { return getRuleContext(IdentifierContext.class,0); } @@ -3736,206 +4367,72 @@ public ExpressionContext expression(int i) { public SetQuantifierContext setQuantifier() { return getRuleContext(SetQuantifierContext.class,0); } - public FunctionCallContext(PrimaryExpressionContext ctx) { copyFrom(ctx); } - @Override - public void enterRule(ParseTreeListener listener) { - if ( listener instanceof SqlBaseListener ) ((SqlBaseListener)listener).enterFunctionCall(this); - } - @Override - public void exitRule(ParseTreeListener listener) { - if ( listener instanceof SqlBaseListener ) ((SqlBaseListener)listener).exitFunctionCall(this); - } - @Override - public T accept(ParseTreeVisitor visitor) { - if ( visitor instanceof SqlBaseVisitor ) return ((SqlBaseVisitor)visitor).visitFunctionCall(this); - else return visitor.visitChildren(this); - } - } - public static class SubqueryExpressionContext extends PrimaryExpressionContext { - public QueryContext query() { - return getRuleContext(QueryContext.class,0); + public FunctionTemplateContext(ParserRuleContext parent, int invokingState) { + super(parent, invokingState); } - public SubqueryExpressionContext(PrimaryExpressionContext ctx) { copyFrom(ctx); } + @Override public int getRuleIndex() { return RULE_functionTemplate; } @Override public void enterRule(ParseTreeListener listener) { - if ( listener instanceof SqlBaseListener ) ((SqlBaseListener)listener).enterSubqueryExpression(this); + if ( listener instanceof SqlBaseListener ) ((SqlBaseListener)listener).enterFunctionTemplate(this); } @Override public void exitRule(ParseTreeListener listener) { - if ( listener instanceof SqlBaseListener ) ((SqlBaseListener)listener).exitSubqueryExpression(this); + if ( listener instanceof SqlBaseListener ) ((SqlBaseListener)listener).exitFunctionTemplate(this); } @Override public T accept(ParseTreeVisitor visitor) { - if ( visitor instanceof SqlBaseVisitor ) return ((SqlBaseVisitor)visitor).visitSubqueryExpression(this); + if ( visitor instanceof SqlBaseVisitor ) return ((SqlBaseVisitor)visitor).visitFunctionTemplate(this); else return visitor.visitChildren(this); } } - public final PrimaryExpressionContext primaryExpression() throws RecognitionException { - PrimaryExpressionContext _localctx = new PrimaryExpressionContext(_ctx, getState()); - enterRule(_localctx, 52, RULE_primaryExpression); + public final FunctionTemplateContext functionTemplate() throws RecognitionException { + FunctionTemplateContext _localctx = new FunctionTemplateContext(_ctx, getState()); + enterRule(_localctx, 68, RULE_functionTemplate); int _la; try { - setState(587); - _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,83,_ctx) ) { - case 1: - _localctx = new CastContext(_localctx); - enterOuterAlt(_localctx, 1); - { - setState(538); - match(CAST); - setState(539); - match(T__0); - setState(540); - expression(); - setState(541); - match(AS); - setState(542); - dataType(); - setState(543); - match(T__1); - } - break; - case 2: - _localctx = new ExtractContext(_localctx); - enterOuterAlt(_localctx, 2); - { - setState(545); - match(EXTRACT); - setState(546); - match(T__0); - setState(547); - ((ExtractContext)_localctx).field = identifier(); - setState(548); - match(FROM); - setState(549); - valueExpression(0); - setState(550); - match(T__1); - } - break; - case 3: - _localctx = new ConstantDefaultContext(_localctx); - enterOuterAlt(_localctx, 3); - { - setState(552); - constant(); - } - break; - case 4: - _localctx = new StarContext(_localctx); - enterOuterAlt(_localctx, 4); - { - setState(553); - match(ASTERISK); - } - break; - case 5: - _localctx = new StarContext(_localctx); - enterOuterAlt(_localctx, 5); + enterOuterAlt(_localctx, 1); + { + setState(625); + identifier(); + setState(626); + match(T__0); + setState(638); + _la = _input.LA(1); + if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << T__0) | (1L << ALL) | (1L << ANALYZE) | (1L << ANALYZED) | (1L << CAST) | (1L << CATALOGS) | (1L << COLUMNS) | (1L << DEBUG) | (1L << DISTINCT) | (1L << EXECUTABLE) | (1L << EXISTS) | (1L << EXPLAIN) | (1L << EXTRACT) | (1L << FALSE) | (1L << FORMAT) | (1L << FUNCTIONS) | (1L << GRAPHVIZ) | (1L << MAPPED) | (1L << MATCH) | (1L << NOT) | (1L << NULL) | (1L << OPTIMIZED) | (1L << PARSED) | (1L << PHYSICAL) | (1L << PLAN) | (1L << RLIKE) | (1L << QUERY) | (1L << SCHEMAS) | (1L << SHOW) | (1L << SYS) | (1L << TABLES))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (TEXT - 64)) | (1L << (TRUE - 64)) | (1L << (TYPE - 64)) | (1L << (TYPES - 64)) | (1L << (VERIFY - 64)) | (1L << (FUNCTION_ESC - 64)) | (1L << (DATE_ESC - 64)) | (1L << (TIME_ESC - 64)) | (1L << (TIMESTAMP_ESC - 64)) | (1L << (GUID_ESC - 64)) | (1L << (PLUS - 64)) | (1L << (MINUS - 64)) | (1L << (ASTERISK - 64)) | (1L << (PARAM - 64)) | (1L << (STRING - 64)) | (1L << (INTEGER_VALUE - 64)) | (1L << (DECIMAL_VALUE - 64)) | (1L << (IDENTIFIER - 64)) | (1L << (DIGIT_IDENTIFIER - 64)) | (1L << (QUOTED_IDENTIFIER - 64)) | (1L << (BACKQUOTED_IDENTIFIER - 64)))) != 0)) { { - setState(557); + setState(628); _la = _input.LA(1); - if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << ANALYZE) | (1L << ANALYZED) | (1L << CATALOGS) | (1L << COLUMNS) | (1L << DEBUG) | (1L << EXECUTABLE) | (1L << EXPLAIN) | (1L << FORMAT) | (1L << FUNCTIONS) | (1L << GRAPHVIZ) | (1L << MAPPED) | (1L << OPTIMIZED) | (1L << PARSED) | (1L << PHYSICAL) | (1L << PLAN) | (1L << RLIKE) | (1L << QUERY) | (1L << SCHEMAS) | (1L << SHOW) | (1L << SYS) | (1L << TABLES))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (TEXT - 64)) | (1L << (TYPE - 64)) | (1L << (TYPES - 64)) | (1L << (VERIFY - 64)) | (1L << (IDENTIFIER - 64)) | (1L << (DIGIT_IDENTIFIER - 64)) | (1L << (QUOTED_IDENTIFIER - 64)) | (1L << (BACKQUOTED_IDENTIFIER - 64)))) != 0)) { + if (_la==ALL || _la==DISTINCT) { { - setState(554); - qualifiedName(); - setState(555); - match(DOT); + setState(627); + setQuantifier(); } } - setState(559); - match(ASTERISK); - } - break; - case 6: - _localctx = new FunctionCallContext(_localctx); - enterOuterAlt(_localctx, 6); - { - setState(560); - identifier(); - setState(561); - match(T__0); - setState(573); + setState(630); + expression(); + setState(635); + _errHandler.sync(this); _la = _input.LA(1); - if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << T__0) | (1L << ALL) | (1L << ANALYZE) | (1L << ANALYZED) | (1L << CAST) | (1L << CATALOGS) | (1L << COLUMNS) | (1L << DEBUG) | (1L << DISTINCT) | (1L << EXECUTABLE) | (1L << EXISTS) | (1L << EXPLAIN) | (1L << EXTRACT) | (1L << FALSE) | (1L << FORMAT) | (1L << FUNCTIONS) | (1L << GRAPHVIZ) | (1L << MAPPED) | (1L << MATCH) | (1L << NOT) | (1L << NULL) | (1L << OPTIMIZED) | (1L << PARSED) | (1L << PHYSICAL) | (1L << PLAN) | (1L << RLIKE) | (1L << QUERY) | (1L << SCHEMAS) | (1L << SHOW) | (1L << SYS) | (1L << TABLES))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (TEXT - 64)) | (1L << (TRUE - 64)) | (1L << (TYPE - 64)) | (1L << (TYPES - 64)) | (1L << (VERIFY - 64)) | (1L << (PLUS - 64)) | (1L << (MINUS - 64)) | (1L << (ASTERISK - 64)) | (1L << (PARAM - 64)) | (1L << (STRING - 64)) | (1L << (INTEGER_VALUE - 64)) | (1L << (DECIMAL_VALUE - 64)) | (1L << (IDENTIFIER - 64)) | (1L << (DIGIT_IDENTIFIER - 64)) | (1L << (QUOTED_IDENTIFIER - 64)) | (1L << (BACKQUOTED_IDENTIFIER - 64)))) != 0)) { + while (_la==T__2) { { - setState(563); - _la = _input.LA(1); - if (_la==ALL || _la==DISTINCT) { - { - setState(562); - setQuantifier(); - } - } - - setState(565); + { + setState(631); + match(T__2); + setState(632); expression(); - setState(570); - _errHandler.sync(this); - _la = _input.LA(1); - while (_la==T__2) { - { - { - setState(566); - match(T__2); - setState(567); - expression(); - } - } - setState(572); - _errHandler.sync(this); - _la = _input.LA(1); } } + setState(637); + _errHandler.sync(this); + _la = _input.LA(1); } - - setState(575); - match(T__1); - } - break; - case 7: - _localctx = new SubqueryExpressionContext(_localctx); - enterOuterAlt(_localctx, 7); - { - setState(577); - match(T__0); - setState(578); - query(); - setState(579); - match(T__1); - } - break; - case 8: - _localctx = new ColumnReferenceContext(_localctx); - enterOuterAlt(_localctx, 8); - { - setState(581); - identifier(); - } - break; - case 9: - _localctx = new DereferenceContext(_localctx); - enterOuterAlt(_localctx, 9); - { - setState(582); - qualifiedName(); - } - break; - case 10: - _localctx = new ParenthesizedExpressionContext(_localctx); - enterOuterAlt(_localctx, 10); - { - setState(583); - match(T__0); - setState(584); - expression(); - setState(585); - match(T__1); } - break; + } + + setState(640); + match(T__1); } } catch (RecognitionException re) { @@ -3977,6 +4474,27 @@ public T accept(ParseTreeVisitor visitor) { else return visitor.visitChildren(this); } } + public static class TimestampEscapedLiteralContext extends ConstantContext { + public TerminalNode TIMESTAMP_ESC() { return getToken(SqlBaseParser.TIMESTAMP_ESC, 0); } + public StringContext string() { + return getRuleContext(StringContext.class,0); + } + public TerminalNode ESC_END() { return getToken(SqlBaseParser.ESC_END, 0); } + public TimestampEscapedLiteralContext(ConstantContext ctx) { copyFrom(ctx); } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof SqlBaseListener ) ((SqlBaseListener)listener).enterTimestampEscapedLiteral(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof SqlBaseListener ) ((SqlBaseListener)listener).exitTimestampEscapedLiteral(this); + } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof SqlBaseVisitor ) return ((SqlBaseVisitor)visitor).visitTimestampEscapedLiteral(this); + else return visitor.visitChildren(this); + } + } public static class StringLiteralContext extends ConstantContext { public List STRING() { return getTokens(SqlBaseParser.STRING); } public TerminalNode STRING(int i) { @@ -4014,6 +4532,48 @@ public T accept(ParseTreeVisitor visitor) { else return visitor.visitChildren(this); } } + public static class TimeEscapedLiteralContext extends ConstantContext { + public TerminalNode TIME_ESC() { return getToken(SqlBaseParser.TIME_ESC, 0); } + public StringContext string() { + return getRuleContext(StringContext.class,0); + } + public TerminalNode ESC_END() { return getToken(SqlBaseParser.ESC_END, 0); } + public TimeEscapedLiteralContext(ConstantContext ctx) { copyFrom(ctx); } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof SqlBaseListener ) ((SqlBaseListener)listener).enterTimeEscapedLiteral(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof SqlBaseListener ) ((SqlBaseListener)listener).exitTimeEscapedLiteral(this); + } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof SqlBaseVisitor ) return ((SqlBaseVisitor)visitor).visitTimeEscapedLiteral(this); + else return visitor.visitChildren(this); + } + } + public static class DateEscapedLiteralContext extends ConstantContext { + public TerminalNode DATE_ESC() { return getToken(SqlBaseParser.DATE_ESC, 0); } + public StringContext string() { + return getRuleContext(StringContext.class,0); + } + public TerminalNode ESC_END() { return getToken(SqlBaseParser.ESC_END, 0); } + public DateEscapedLiteralContext(ConstantContext ctx) { copyFrom(ctx); } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof SqlBaseListener ) ((SqlBaseListener)listener).enterDateEscapedLiteral(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof SqlBaseListener ) ((SqlBaseListener)listener).exitDateEscapedLiteral(this); + } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof SqlBaseVisitor ) return ((SqlBaseVisitor)visitor).visitDateEscapedLiteral(this); + else return visitor.visitChildren(this); + } + } public static class NumericLiteralContext extends ConstantContext { public NumberContext number() { return getRuleContext(NumberContext.class,0); @@ -4052,19 +4612,40 @@ public T accept(ParseTreeVisitor visitor) { else return visitor.visitChildren(this); } } + public static class GuidEscapedLiteralContext extends ConstantContext { + public TerminalNode GUID_ESC() { return getToken(SqlBaseParser.GUID_ESC, 0); } + public StringContext string() { + return getRuleContext(StringContext.class,0); + } + public TerminalNode ESC_END() { return getToken(SqlBaseParser.ESC_END, 0); } + public GuidEscapedLiteralContext(ConstantContext ctx) { copyFrom(ctx); } + @Override + public void enterRule(ParseTreeListener listener) { + if ( listener instanceof SqlBaseListener ) ((SqlBaseListener)listener).enterGuidEscapedLiteral(this); + } + @Override + public void exitRule(ParseTreeListener listener) { + if ( listener instanceof SqlBaseListener ) ((SqlBaseListener)listener).exitGuidEscapedLiteral(this); + } + @Override + public T accept(ParseTreeVisitor visitor) { + if ( visitor instanceof SqlBaseVisitor ) return ((SqlBaseVisitor)visitor).visitGuidEscapedLiteral(this); + else return visitor.visitChildren(this); + } + } public final ConstantContext constant() throws RecognitionException { ConstantContext _localctx = new ConstantContext(_ctx, getState()); - enterRule(_localctx, 54, RULE_constant); + enterRule(_localctx, 70, RULE_constant); try { int _alt; - setState(598); + setState(667); switch (_input.LA(1)) { case NULL: _localctx = new NullLiteralContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(589); + setState(642); match(NULL); } break; @@ -4073,7 +4654,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new NumericLiteralContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(590); + setState(643); number(); } break; @@ -4082,7 +4663,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new BooleanLiteralContext(_localctx); enterOuterAlt(_localctx, 3); { - setState(591); + setState(644); booleanValue(); } break; @@ -4090,7 +4671,7 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new StringLiteralContext(_localctx); enterOuterAlt(_localctx, 4); { - setState(593); + setState(646); _errHandler.sync(this); _alt = 1; do { @@ -4098,7 +4679,7 @@ public final ConstantContext constant() throws RecognitionException { case 1: { { - setState(592); + setState(645); match(STRING); } } @@ -4106,9 +4687,9 @@ public final ConstantContext constant() throws RecognitionException { default: throw new NoViableAltException(this); } - setState(595); + setState(648); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,84,_ctx); + _alt = getInterpreter().adaptivePredict(_input,89,_ctx); } while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ); } break; @@ -4116,10 +4697,58 @@ public final ConstantContext constant() throws RecognitionException { _localctx = new ParamLiteralContext(_localctx); enterOuterAlt(_localctx, 5); { - setState(597); + setState(650); match(PARAM); } break; + case DATE_ESC: + _localctx = new DateEscapedLiteralContext(_localctx); + enterOuterAlt(_localctx, 6); + { + setState(651); + match(DATE_ESC); + setState(652); + string(); + setState(653); + match(ESC_END); + } + break; + case TIME_ESC: + _localctx = new TimeEscapedLiteralContext(_localctx); + enterOuterAlt(_localctx, 7); + { + setState(655); + match(TIME_ESC); + setState(656); + string(); + setState(657); + match(ESC_END); + } + break; + case TIMESTAMP_ESC: + _localctx = new TimestampEscapedLiteralContext(_localctx); + enterOuterAlt(_localctx, 8); + { + setState(659); + match(TIMESTAMP_ESC); + setState(660); + string(); + setState(661); + match(ESC_END); + } + break; + case GUID_ESC: + _localctx = new GuidEscapedLiteralContext(_localctx); + enterOuterAlt(_localctx, 9); + { + setState(663); + match(GUID_ESC); + setState(664); + string(); + setState(665); + match(ESC_END); + } + break; default: throw new NoViableAltException(this); } @@ -4163,14 +4792,14 @@ public T accept(ParseTreeVisitor visitor) { public final ComparisonOperatorContext comparisonOperator() throws RecognitionException { ComparisonOperatorContext _localctx = new ComparisonOperatorContext(_ctx, getState()); - enterRule(_localctx, 56, RULE_comparisonOperator); + enterRule(_localctx, 72, RULE_comparisonOperator); int _la; try { enterOuterAlt(_localctx, 1); { - setState(600); + setState(669); _la = _input.LA(1); - if ( !(((((_la - 72)) & ~0x3f) == 0 && ((1L << (_la - 72)) & ((1L << (EQ - 72)) | (1L << (NEQ - 72)) | (1L << (LT - 72)) | (1L << (LTE - 72)) | (1L << (GT - 72)) | (1L << (GTE - 72)))) != 0)) ) { + if ( !(((((_la - 80)) & ~0x3f) == 0 && ((1L << (_la - 80)) & ((1L << (EQ - 80)) | (1L << (NEQ - 80)) | (1L << (LT - 80)) | (1L << (LTE - 80)) | (1L << (GT - 80)) | (1L << (GTE - 80)))) != 0)) ) { _errHandler.recoverInline(this); } else { consume(); @@ -4212,12 +4841,12 @@ public T accept(ParseTreeVisitor visitor) { public final BooleanValueContext booleanValue() throws RecognitionException { BooleanValueContext _localctx = new BooleanValueContext(_ctx, getState()); - enterRule(_localctx, 58, RULE_booleanValue); + enterRule(_localctx, 74, RULE_booleanValue); int _la; try { enterOuterAlt(_localctx, 1); { - setState(602); + setState(671); _la = _input.LA(1); if ( !(_la==FALSE || _la==TRUE) ) { _errHandler.recoverInline(this); @@ -4270,12 +4899,12 @@ public T accept(ParseTreeVisitor visitor) { public final DataTypeContext dataType() throws RecognitionException { DataTypeContext _localctx = new DataTypeContext(_ctx, getState()); - enterRule(_localctx, 60, RULE_dataType); + enterRule(_localctx, 76, RULE_dataType); try { _localctx = new PrimitiveDataTypeContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(604); + setState(673); identifier(); } } @@ -4322,30 +4951,30 @@ public T accept(ParseTreeVisitor visitor) { public final QualifiedNameContext qualifiedName() throws RecognitionException { QualifiedNameContext _localctx = new QualifiedNameContext(_ctx, getState()); - enterRule(_localctx, 62, RULE_qualifiedName); + enterRule(_localctx, 78, RULE_qualifiedName); try { int _alt; enterOuterAlt(_localctx, 1); { - setState(611); + setState(680); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,86,_ctx); + _alt = getInterpreter().adaptivePredict(_input,91,_ctx); while ( _alt!=2 && _alt!=org.antlr.v4.runtime.atn.ATN.INVALID_ALT_NUMBER ) { if ( _alt==1 ) { { { - setState(606); + setState(675); identifier(); - setState(607); + setState(676); match(DOT); } } } - setState(613); + setState(682); _errHandler.sync(this); - _alt = getInterpreter().adaptivePredict(_input,86,_ctx); + _alt = getInterpreter().adaptivePredict(_input,91,_ctx); } - setState(614); + setState(683); identifier(); } } @@ -4388,15 +5017,15 @@ public T accept(ParseTreeVisitor visitor) { public final IdentifierContext identifier() throws RecognitionException { IdentifierContext _localctx = new IdentifierContext(_ctx, getState()); - enterRule(_localctx, 64, RULE_identifier); + enterRule(_localctx, 80, RULE_identifier); try { - setState(618); + setState(687); switch (_input.LA(1)) { case QUOTED_IDENTIFIER: case BACKQUOTED_IDENTIFIER: enterOuterAlt(_localctx, 1); { - setState(616); + setState(685); quoteIdentifier(); } break; @@ -4429,7 +5058,7 @@ public final IdentifierContext identifier() throws RecognitionException { case DIGIT_IDENTIFIER: enterOuterAlt(_localctx, 2); { - setState(617); + setState(686); unquoteIdentifier(); } break; @@ -4479,46 +5108,46 @@ public T accept(ParseTreeVisitor visitor) { public final TableIdentifierContext tableIdentifier() throws RecognitionException { TableIdentifierContext _localctx = new TableIdentifierContext(_ctx, getState()); - enterRule(_localctx, 66, RULE_tableIdentifier); + enterRule(_localctx, 82, RULE_tableIdentifier); int _la; try { - setState(632); + setState(701); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,90,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,95,_ctx) ) { case 1: enterOuterAlt(_localctx, 1); { - setState(623); + setState(692); _la = _input.LA(1); if ((((_la) & ~0x3f) == 0 && ((1L << _la) & ((1L << ANALYZE) | (1L << ANALYZED) | (1L << CATALOGS) | (1L << COLUMNS) | (1L << DEBUG) | (1L << EXECUTABLE) | (1L << EXPLAIN) | (1L << FORMAT) | (1L << FUNCTIONS) | (1L << GRAPHVIZ) | (1L << MAPPED) | (1L << OPTIMIZED) | (1L << PARSED) | (1L << PHYSICAL) | (1L << PLAN) | (1L << RLIKE) | (1L << QUERY) | (1L << SCHEMAS) | (1L << SHOW) | (1L << SYS) | (1L << TABLES))) != 0) || ((((_la - 64)) & ~0x3f) == 0 && ((1L << (_la - 64)) & ((1L << (TEXT - 64)) | (1L << (TYPE - 64)) | (1L << (TYPES - 64)) | (1L << (VERIFY - 64)) | (1L << (IDENTIFIER - 64)) | (1L << (DIGIT_IDENTIFIER - 64)) | (1L << (QUOTED_IDENTIFIER - 64)) | (1L << (BACKQUOTED_IDENTIFIER - 64)))) != 0)) { { - setState(620); + setState(689); ((TableIdentifierContext)_localctx).catalog = identifier(); - setState(621); + setState(690); match(T__3); } } - setState(625); + setState(694); match(TABLE_IDENTIFIER); } break; case 2: enterOuterAlt(_localctx, 2); { - setState(629); + setState(698); _errHandler.sync(this); - switch ( getInterpreter().adaptivePredict(_input,89,_ctx) ) { + switch ( getInterpreter().adaptivePredict(_input,94,_ctx) ) { case 1: { - setState(626); + setState(695); ((TableIdentifierContext)_localctx).catalog = identifier(); - setState(627); + setState(696); match(T__3); } break; } - setState(631); + setState(700); ((TableIdentifierContext)_localctx).name = identifier(); } break; @@ -4583,15 +5212,15 @@ public T accept(ParseTreeVisitor visitor) { public final QuoteIdentifierContext quoteIdentifier() throws RecognitionException { QuoteIdentifierContext _localctx = new QuoteIdentifierContext(_ctx, getState()); - enterRule(_localctx, 68, RULE_quoteIdentifier); + enterRule(_localctx, 84, RULE_quoteIdentifier); try { - setState(636); + setState(705); switch (_input.LA(1)) { case QUOTED_IDENTIFIER: _localctx = new QuotedIdentifierContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(634); + setState(703); match(QUOTED_IDENTIFIER); } break; @@ -4599,7 +5228,7 @@ public final QuoteIdentifierContext quoteIdentifier() throws RecognitionExceptio _localctx = new BackQuotedIdentifierContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(635); + setState(704); match(BACKQUOTED_IDENTIFIER); } break; @@ -4669,15 +5298,15 @@ public T accept(ParseTreeVisitor visitor) { public final UnquoteIdentifierContext unquoteIdentifier() throws RecognitionException { UnquoteIdentifierContext _localctx = new UnquoteIdentifierContext(_ctx, getState()); - enterRule(_localctx, 70, RULE_unquoteIdentifier); + enterRule(_localctx, 86, RULE_unquoteIdentifier); try { - setState(641); + setState(710); switch (_input.LA(1)) { case IDENTIFIER: _localctx = new UnquotedIdentifierContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(638); + setState(707); match(IDENTIFIER); } break; @@ -4709,7 +5338,7 @@ public final UnquoteIdentifierContext unquoteIdentifier() throws RecognitionExce _localctx = new UnquotedIdentifierContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(639); + setState(708); nonReserved(); } break; @@ -4717,7 +5346,7 @@ public final UnquoteIdentifierContext unquoteIdentifier() throws RecognitionExce _localctx = new DigitIdentifierContext(_localctx); enterOuterAlt(_localctx, 3); { - setState(640); + setState(709); match(DIGIT_IDENTIFIER); } break; @@ -4784,15 +5413,15 @@ public T accept(ParseTreeVisitor visitor) { public final NumberContext number() throws RecognitionException { NumberContext _localctx = new NumberContext(_ctx, getState()); - enterRule(_localctx, 72, RULE_number); + enterRule(_localctx, 88, RULE_number); try { - setState(645); + setState(714); switch (_input.LA(1)) { case DECIMAL_VALUE: _localctx = new DecimalLiteralContext(_localctx); enterOuterAlt(_localctx, 1); { - setState(643); + setState(712); match(DECIMAL_VALUE); } break; @@ -4800,7 +5429,7 @@ public final NumberContext number() throws RecognitionException { _localctx = new IntegerLiteralContext(_localctx); enterOuterAlt(_localctx, 2); { - setState(644); + setState(713); match(INTEGER_VALUE); } break; @@ -4843,12 +5472,12 @@ public T accept(ParseTreeVisitor visitor) { public final StringContext string() throws RecognitionException { StringContext _localctx = new StringContext(_ctx, getState()); - enterRule(_localctx, 74, RULE_string); + enterRule(_localctx, 90, RULE_string); int _la; try { enterOuterAlt(_localctx, 1); { - setState(647); + setState(716); _la = _input.LA(1); if ( !(_la==PARAM || _la==STRING) ) { _errHandler.recoverInline(this); @@ -4915,12 +5544,12 @@ public T accept(ParseTreeVisitor visitor) { public final NonReservedContext nonReserved() throws RecognitionException { NonReservedContext _localctx = new NonReservedContext(_ctx, getState()); - enterRule(_localctx, 76, RULE_nonReserved); + enterRule(_localctx, 92, RULE_nonReserved); int _la; try { enterOuterAlt(_localctx, 1); { - setState(649); + setState(718); _la = _input.LA(1); if ( !(((((_la - 6)) & ~0x3f) == 0 && ((1L << (_la - 6)) & ((1L << (ANALYZE - 6)) | (1L << (ANALYZED - 6)) | (1L << (CATALOGS - 6)) | (1L << (COLUMNS - 6)) | (1L << (DEBUG - 6)) | (1L << (EXECUTABLE - 6)) | (1L << (EXPLAIN - 6)) | (1L << (FORMAT - 6)) | (1L << (FUNCTIONS - 6)) | (1L << (GRAPHVIZ - 6)) | (1L << (MAPPED - 6)) | (1L << (OPTIMIZED - 6)) | (1L << (PARSED - 6)) | (1L << (PHYSICAL - 6)) | (1L << (PLAN - 6)) | (1L << (RLIKE - 6)) | (1L << (QUERY - 6)) | (1L << (SCHEMAS - 6)) | (1L << (SHOW - 6)) | (1L << (SYS - 6)) | (1L << (TABLES - 6)) | (1L << (TEXT - 6)) | (1L << (TYPE - 6)) | (1L << (TYPES - 6)) | (1L << (VERIFY - 6)))) != 0)) ) { _errHandler.recoverInline(this); @@ -4942,9 +5571,9 @@ public final NonReservedContext nonReserved() throws RecognitionException { public boolean sempred(RuleContext _localctx, int ruleIndex, int predIndex) { switch (ruleIndex) { - case 21: + case 22: return booleanExpression_sempred((BooleanExpressionContext)_localctx, predIndex); - case 25: + case 27: return valueExpression_sempred((ValueExpressionContext)_localctx, predIndex); } return true; @@ -4971,266 +5600,291 @@ private boolean valueExpression_sempred(ValueExpressionContext _localctx, int pr } public static final String _serializedATN = - "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3d\u028e\4\2\t\2\4"+ + "\3\u0430\ud6d1\u8206\uad2d\u4417\uaef1\u8d80\uaadd\3l\u02d3\4\2\t\2\4"+ "\3\t\3\4\4\t\4\4\5\t\5\4\6\t\6\4\7\t\7\4\b\t\b\4\t\t\t\4\n\t\n\4\13\t"+ "\13\4\f\t\f\4\r\t\r\4\16\t\16\4\17\t\17\4\20\t\20\4\21\t\21\4\22\t\22"+ "\4\23\t\23\4\24\t\24\4\25\t\25\4\26\t\26\4\27\t\27\4\30\t\30\4\31\t\31"+ "\4\32\t\32\4\33\t\33\4\34\t\34\4\35\t\35\4\36\t\36\4\37\t\37\4 \t \4!"+ - "\t!\4\"\t\"\4#\t#\4$\t$\4%\t%\4&\t&\4\'\t\'\4(\t(\3\2\3\2\3\2\3\3\3\3"+ - "\3\3\3\4\3\4\3\4\3\4\3\4\3\4\3\4\3\4\3\4\7\4`\n\4\f\4\16\4c\13\4\3\4\5"+ - "\4f\n\4\3\4\3\4\3\4\3\4\3\4\3\4\3\4\7\4o\n\4\f\4\16\4r\13\4\3\4\5\4u\n"+ - "\4\3\4\3\4\3\4\3\4\5\4{\n\4\3\4\5\4~\n\4\3\4\3\4\3\4\3\4\3\4\3\4\3\4\3"+ - "\4\3\4\5\4\u0089\n\4\3\4\5\4\u008c\n\4\3\4\3\4\3\4\3\4\3\4\3\4\3\4\3\4"+ - "\5\4\u0096\n\4\3\4\5\4\u0099\n\4\3\4\5\4\u009c\n\4\3\4\5\4\u009f\n\4\3"+ - "\4\3\4\3\4\3\4\7\4\u00a5\n\4\f\4\16\4\u00a8\13\4\5\4\u00aa\n\4\3\4\3\4"+ - "\3\4\3\4\5\4\u00b0\n\4\3\4\3\4\5\4\u00b4\n\4\3\4\5\4\u00b7\n\4\3\4\5\4"+ - "\u00ba\n\4\3\4\5\4\u00bd\n\4\3\4\3\4\3\4\3\4\3\4\5\4\u00c4\n\4\3\5\3\5"+ - "\3\5\3\5\7\5\u00ca\n\5\f\5\16\5\u00cd\13\5\5\5\u00cf\n\5\3\5\3\5\3\6\3"+ - "\6\3\6\3\6\3\6\3\6\7\6\u00d9\n\6\f\6\16\6\u00dc\13\6\5\6\u00de\n\6\3\6"+ - "\3\6\5\6\u00e2\n\6\3\7\3\7\3\7\3\7\3\7\5\7\u00e9\n\7\3\b\3\b\5\b\u00ed"+ - "\n\b\3\t\3\t\5\t\u00f1\n\t\3\t\3\t\3\t\7\t\u00f6\n\t\f\t\16\t\u00f9\13"+ - "\t\3\t\5\t\u00fc\n\t\3\t\3\t\5\t\u0100\n\t\3\t\3\t\3\t\5\t\u0105\n\t\3"+ - "\t\3\t\5\t\u0109\n\t\3\n\3\n\3\n\3\n\7\n\u010f\n\n\f\n\16\n\u0112\13\n"+ - "\3\13\5\13\u0115\n\13\3\13\3\13\3\13\7\13\u011a\n\13\f\13\16\13\u011d"+ - "\13\13\3\f\3\f\3\r\3\r\3\r\3\r\7\r\u0125\n\r\f\r\16\r\u0128\13\r\5\r\u012a"+ - "\n\r\3\r\3\r\5\r\u012e\n\r\3\16\3\16\3\16\3\16\3\16\3\16\3\17\3\17\3\20"+ - "\3\20\5\20\u013a\n\20\3\20\5\20\u013d\n\20\3\21\3\21\7\21\u0141\n\21\f"+ - "\21\16\21\u0144\13\21\3\22\3\22\3\22\3\22\5\22\u014a\n\22\3\22\3\22\3"+ - "\22\3\22\3\22\5\22\u0151\n\22\3\23\5\23\u0154\n\23\3\23\3\23\5\23\u0158"+ - "\n\23\3\23\3\23\5\23\u015c\n\23\3\23\3\23\5\23\u0160\n\23\5\23\u0162\n"+ - "\23\3\24\3\24\3\24\3\24\3\24\3\24\3\24\7\24\u016b\n\24\f\24\16\24\u016e"+ - "\13\24\3\24\3\24\5\24\u0172\n\24\3\25\3\25\5\25\u0176\n\25\3\25\5\25\u0179"+ - "\n\25\3\25\3\25\3\25\3\25\5\25\u017f\n\25\3\25\5\25\u0182\n\25\3\25\3"+ - "\25\3\25\3\25\5\25\u0188\n\25\3\25\5\25\u018b\n\25\5\25\u018d\n\25\3\26"+ - "\3\26\3\27\3\27\3\27\3\27\3\27\3\27\3\27\3\27\3\27\3\27\3\27\3\27\3\27"+ - "\7\27\u019e\n\27\f\27\16\27\u01a1\13\27\3\27\3\27\3\27\3\27\3\27\3\27"+ - "\3\27\3\27\3\27\7\27\u01ac\n\27\f\27\16\27\u01af\13\27\3\27\3\27\3\27"+ - "\3\27\3\27\3\27\3\27\3\27\3\27\7\27\u01ba\n\27\f\27\16\27\u01bd\13\27"+ - "\3\27\3\27\3\27\5\27\u01c2\n\27\3\27\3\27\3\27\3\27\3\27\3\27\7\27\u01ca"+ - "\n\27\f\27\16\27\u01cd\13\27\3\30\3\30\5\30\u01d1\n\30\3\31\5\31\u01d4"+ - "\n\31\3\31\3\31\3\31\3\31\3\31\3\31\5\31\u01dc\n\31\3\31\3\31\3\31\3\31"+ - "\3\31\7\31\u01e3\n\31\f\31\16\31\u01e6\13\31\3\31\3\31\3\31\5\31\u01eb"+ - "\n\31\3\31\3\31\3\31\3\31\3\31\3\31\5\31\u01f3\n\31\3\31\3\31\3\31\5\31"+ - "\u01f8\n\31\3\31\3\31\3\31\3\31\5\31\u01fe\n\31\3\31\5\31\u0201\n\31\3"+ - "\32\3\32\3\32\5\32\u0206\n\32\3\33\3\33\3\33\3\33\5\33\u020c\n\33\3\33"+ - "\3\33\3\33\3\33\3\33\3\33\3\33\3\33\3\33\3\33\7\33\u0218\n\33\f\33\16"+ - "\33\u021b\13\33\3\34\3\34\3\34\3\34\3\34\3\34\3\34\3\34\3\34\3\34\3\34"+ - "\3\34\3\34\3\34\3\34\3\34\3\34\3\34\3\34\5\34\u0230\n\34\3\34\3\34\3\34"+ - "\3\34\5\34\u0236\n\34\3\34\3\34\3\34\7\34\u023b\n\34\f\34\16\34\u023e"+ - "\13\34\5\34\u0240\n\34\3\34\3\34\3\34\3\34\3\34\3\34\3\34\3\34\3\34\3"+ - "\34\3\34\3\34\5\34\u024e\n\34\3\35\3\35\3\35\3\35\6\35\u0254\n\35\r\35"+ - "\16\35\u0255\3\35\5\35\u0259\n\35\3\36\3\36\3\37\3\37\3 \3 \3!\3!\3!\7"+ - "!\u0264\n!\f!\16!\u0267\13!\3!\3!\3\"\3\"\5\"\u026d\n\"\3#\3#\3#\5#\u0272"+ - "\n#\3#\3#\3#\3#\5#\u0278\n#\3#\5#\u027b\n#\3$\3$\5$\u027f\n$\3%\3%\3%"+ - "\5%\u0284\n%\3&\3&\5&\u0288\n&\3\'\3\'\3(\3(\3(\2\4,\64)\2\4\6\b\n\f\16"+ - "\20\22\24\26\30\32\34\36 \"$&(*,.\60\62\64\668:<>@BDFHJLN\2\20\b\2\7\7"+ - "\t\t\31\31,,\62\62\66\66\4\2\"\"BB\4\2\t\t\62\62\4\2\37\37%%\3\2\25\26"+ - "\4\2\7\7YY\4\2\r\r\25\25\4\2\7\7\27\27\3\2PQ\3\2RT\3\2JO\4\2\35\35CC\3"+ - "\2WX\20\2\b\t\22\24\31\31\33\33\36\36!\",,\62\62\668:<>?ABDEGG\u02e8\2"+ - "P\3\2\2\2\4S\3\2\2\2\6\u00c3\3\2\2\2\b\u00ce\3\2\2\2\n\u00d2\3\2\2\2\f"+ - "\u00e8\3\2\2\2\16\u00ea\3\2\2\2\20\u00ee\3\2\2\2\22\u010a\3\2\2\2\24\u0114"+ - "\3\2\2\2\26\u011e\3\2\2\2\30\u012d\3\2\2\2\32\u012f\3\2\2\2\34\u0135\3"+ - "\2\2\2\36\u0137\3\2\2\2 \u013e\3\2\2\2\"\u0150\3\2\2\2$\u0161\3\2\2\2"+ - "&\u0171\3\2\2\2(\u018c\3\2\2\2*\u018e\3\2\2\2,\u01c1\3\2\2\2.\u01ce\3"+ - "\2\2\2\60\u0200\3\2\2\2\62\u0202\3\2\2\2\64\u020b\3\2\2\2\66\u024d\3\2"+ - "\2\28\u0258\3\2\2\2:\u025a\3\2\2\2<\u025c\3\2\2\2>\u025e\3\2\2\2@\u0265"+ - "\3\2\2\2B\u026c\3\2\2\2D\u027a\3\2\2\2F\u027e\3\2\2\2H\u0283\3\2\2\2J"+ - "\u0287\3\2\2\2L\u0289\3\2\2\2N\u028b\3\2\2\2PQ\5\6\4\2QR\7\2\2\3R\3\3"+ - "\2\2\2ST\5*\26\2TU\7\2\2\3U\5\3\2\2\2V\u00c4\5\b\5\2We\7\33\2\2Xa\7\3"+ - "\2\2YZ\78\2\2Z`\t\2\2\2[\\\7\36\2\2\\`\t\3\2\2]^\7G\2\2^`\5<\37\2_Y\3"+ - "\2\2\2_[\3\2\2\2_]\3\2\2\2`c\3\2\2\2a_\3\2\2\2ab\3\2\2\2bd\3\2\2\2ca\3"+ - "\2\2\2df\7\4\2\2eX\3\2\2\2ef\3\2\2\2fg\3\2\2\2g\u00c4\5\6\4\2ht\7\24\2"+ - "\2ip\7\3\2\2jk\78\2\2ko\t\4\2\2lm\7\36\2\2mo\t\3\2\2nj\3\2\2\2nl\3\2\2"+ - "\2or\3\2\2\2pn\3\2\2\2pq\3\2\2\2qs\3\2\2\2rp\3\2\2\2su\7\4\2\2ti\3\2\2"+ - "\2tu\3\2\2\2uv\3\2\2\2v\u00c4\5\6\4\2wx\7>\2\2x}\7A\2\2y{\7*\2\2zy\3\2"+ - "\2\2z{\3\2\2\2{|\3\2\2\2|~\5\62\32\2}z\3\2\2\2}~\3\2\2\2~\u00c4\3\2\2"+ - "\2\177\u0080\7>\2\2\u0080\u0081\7\23\2\2\u0081\u0082\t\5\2\2\u0082\u00c4"+ - "\5D#\2\u0083\u0084\t\6\2\2\u0084\u00c4\5D#\2\u0085\u0086\7>\2\2\u0086"+ - "\u008b\7!\2\2\u0087\u0089\7*\2\2\u0088\u0087\3\2\2\2\u0088\u0089\3\2\2"+ - "\2\u0089\u008a\3\2\2\2\u008a\u008c\5\62\32\2\u008b\u0088\3\2\2\2\u008b"+ - "\u008c\3\2\2\2\u008c\u00c4\3\2\2\2\u008d\u008e\7>\2\2\u008e\u00c4\7<\2"+ - "\2\u008f\u0090\7?\2\2\u0090\u00c4\7\22\2\2\u0091\u0092\7?\2\2\u0092\u0098"+ - "\7A\2\2\u0093\u0095\7\21\2\2\u0094\u0096\7*\2\2\u0095\u0094\3\2\2\2\u0095"+ - "\u0096\3\2\2\2\u0096\u0097\3\2\2\2\u0097\u0099\5\62\32\2\u0098\u0093\3"+ - "\2\2\2\u0098\u0099\3\2\2\2\u0099\u009e\3\2\2\2\u009a\u009c\7*\2\2\u009b"+ - "\u009a\3\2\2\2\u009b\u009c\3\2\2\2\u009c\u009d\3\2\2\2\u009d\u009f\5\62"+ - "\32\2\u009e\u009b\3\2\2\2\u009e\u009f\3\2\2\2\u009f\u00a9\3\2\2\2\u00a0"+ - "\u00a1\7D\2\2\u00a1\u00a6\5L\'\2\u00a2\u00a3\7\5\2\2\u00a3\u00a5\5L\'"+ - "\2\u00a4\u00a2\3\2\2\2\u00a5\u00a8\3\2\2\2\u00a6\u00a4\3\2\2\2\u00a6\u00a7"+ - "\3\2\2\2\u00a7\u00aa\3\2\2\2\u00a8\u00a6\3\2\2\2\u00a9\u00a0\3\2\2\2\u00a9"+ - "\u00aa\3\2\2\2\u00aa\u00c4\3\2\2\2\u00ab\u00ac\7?\2\2\u00ac\u00af\7\23"+ - "\2\2\u00ad\u00ae\7\21\2\2\u00ae\u00b0\5L\'\2\u00af\u00ad\3\2\2\2\u00af"+ - "\u00b0\3\2\2\2\u00b0\u00b6\3\2\2\2\u00b1\u00b3\7@\2\2\u00b2\u00b4\7*\2"+ - "\2\u00b3\u00b2\3\2\2\2\u00b3\u00b4\3\2\2\2\u00b4\u00b5\3\2\2\2\u00b5\u00b7"+ - "\5\62\32\2\u00b6\u00b1\3\2\2\2\u00b6\u00b7\3\2\2\2\u00b7\u00bc\3\2\2\2"+ - "\u00b8\u00ba\7*\2\2\u00b9\u00b8\3\2\2\2\u00b9\u00ba\3\2\2\2\u00ba\u00bb"+ - "\3\2\2\2\u00bb\u00bd\5\62\32\2\u00bc\u00b9\3\2\2\2\u00bc\u00bd\3\2\2\2"+ - "\u00bd\u00c4\3\2\2\2\u00be\u00bf\7?\2\2\u00bf\u00c4\7E\2\2\u00c0\u00c1"+ - "\7?\2\2\u00c1\u00c2\7@\2\2\u00c2\u00c4\7E\2\2\u00c3V\3\2\2\2\u00c3W\3"+ - "\2\2\2\u00c3h\3\2\2\2\u00c3w\3\2\2\2\u00c3\177\3\2\2\2\u00c3\u0083\3\2"+ - "\2\2\u00c3\u0085\3\2\2\2\u00c3\u008d\3\2\2\2\u00c3\u008f\3\2\2\2\u00c3"+ - "\u0091\3\2\2\2\u00c3\u00ab\3\2\2\2\u00c3\u00be\3\2\2\2\u00c3\u00c0\3\2"+ - "\2\2\u00c4\7\3\2\2\2\u00c5\u00c6\7I\2\2\u00c6\u00cb\5\32\16\2\u00c7\u00c8"+ - "\7\5\2\2\u00c8\u00ca\5\32\16\2\u00c9\u00c7\3\2\2\2\u00ca\u00cd\3\2\2\2"+ - "\u00cb\u00c9\3\2\2\2\u00cb\u00cc\3\2\2\2\u00cc\u00cf\3\2\2\2\u00cd\u00cb"+ - "\3\2\2\2\u00ce\u00c5\3\2\2\2\u00ce\u00cf\3\2\2\2\u00cf\u00d0\3\2\2\2\u00d0"+ - "\u00d1\5\n\6\2\u00d1\t\3\2\2\2\u00d2\u00dd\5\f\7\2\u00d3\u00d4\7\64\2"+ - "\2\u00d4\u00d5\7\17\2\2\u00d5\u00da\5\16\b\2\u00d6\u00d7\7\5\2\2\u00d7"+ - "\u00d9\5\16\b\2\u00d8\u00d6\3\2\2\2\u00d9\u00dc\3\2\2\2\u00da\u00d8\3"+ - "\2\2\2\u00da\u00db\3\2\2\2\u00db\u00de\3\2\2\2\u00dc\u00da\3\2\2\2\u00dd"+ - "\u00d3\3\2\2\2\u00dd\u00de\3\2\2\2\u00de\u00e1\3\2\2\2\u00df\u00e0\7+"+ - "\2\2\u00e0\u00e2\t\7\2\2\u00e1\u00df\3\2\2\2\u00e1\u00e2\3\2\2\2\u00e2"+ - "\13\3\2\2\2\u00e3\u00e9\5\20\t\2\u00e4\u00e5\7\3\2\2\u00e5\u00e6\5\n\6"+ - "\2\u00e6\u00e7\7\4\2\2\u00e7\u00e9\3\2\2\2\u00e8\u00e3\3\2\2\2\u00e8\u00e4"+ - "\3\2\2\2\u00e9\r\3\2\2\2\u00ea\u00ec\5*\26\2\u00eb\u00ed\t\b\2\2\u00ec"+ - "\u00eb\3\2\2\2\u00ec\u00ed\3\2\2\2\u00ed\17\3\2\2\2\u00ee\u00f0\7=\2\2"+ - "\u00ef\u00f1\5\34\17\2\u00f0\u00ef\3\2\2\2\u00f0\u00f1\3\2\2\2\u00f1\u00f2"+ - "\3\2\2\2\u00f2\u00f7\5\36\20\2\u00f3\u00f4\7\5\2\2\u00f4\u00f6\5\36\20"+ - "\2\u00f5\u00f3\3\2\2\2\u00f6\u00f9\3\2\2\2\u00f7\u00f5\3\2\2\2\u00f7\u00f8"+ - "\3\2\2\2\u00f8\u00fb\3\2\2\2\u00f9\u00f7\3\2\2\2\u00fa\u00fc\5\22\n\2"+ - "\u00fb\u00fa\3\2\2\2\u00fb\u00fc\3\2\2\2\u00fc\u00ff\3\2\2\2\u00fd\u00fe"+ - "\7H\2\2\u00fe\u0100\5,\27\2\u00ff\u00fd\3\2\2\2\u00ff\u0100\3\2\2\2\u0100"+ - "\u0104\3\2\2\2\u0101\u0102\7#\2\2\u0102\u0103\7\17\2\2\u0103\u0105\5\24"+ - "\13\2\u0104\u0101\3\2\2\2\u0104\u0105\3\2\2\2\u0105\u0108\3\2\2\2\u0106"+ - "\u0107\7$\2\2\u0107\u0109\5,\27\2\u0108\u0106\3\2\2\2\u0108\u0109\3\2"+ - "\2\2\u0109\21\3\2\2\2\u010a\u010b\7\37\2\2\u010b\u0110\5 \21\2\u010c\u010d"+ - "\7\5\2\2\u010d\u010f\5 \21\2\u010e\u010c\3\2\2\2\u010f\u0112\3\2\2\2\u0110"+ - "\u010e\3\2\2\2\u0110\u0111\3\2\2\2\u0111\23\3\2\2\2\u0112\u0110\3\2\2"+ - "\2\u0113\u0115\5\34\17\2\u0114\u0113\3\2\2\2\u0114\u0115\3\2\2\2\u0115"+ - "\u0116\3\2\2\2\u0116\u011b\5\26\f\2\u0117\u0118\7\5\2\2\u0118\u011a\5"+ - "\26\f\2\u0119\u0117\3\2\2\2\u011a\u011d\3\2\2\2\u011b\u0119\3\2\2\2\u011b"+ - "\u011c\3\2\2\2\u011c\25\3\2\2\2\u011d\u011b\3\2\2\2\u011e\u011f\5\30\r"+ - "\2\u011f\27\3\2\2\2\u0120\u0129\7\3\2\2\u0121\u0126\5*\26\2\u0122\u0123"+ - "\7\5\2\2\u0123\u0125\5*\26\2\u0124\u0122\3\2\2\2\u0125\u0128\3\2\2\2\u0126"+ - "\u0124\3\2\2\2\u0126\u0127\3\2\2\2\u0127\u012a\3\2\2\2\u0128\u0126\3\2"+ - "\2\2\u0129\u0121\3\2\2\2\u0129\u012a\3\2\2\2\u012a\u012b\3\2\2\2\u012b"+ - "\u012e\7\4\2\2\u012c\u012e\5*\26\2\u012d\u0120\3\2\2\2\u012d\u012c\3\2"+ - "\2\2\u012e\31\3\2\2\2\u012f\u0130\5B\"\2\u0130\u0131\7\f\2\2\u0131\u0132"+ - "\7\3\2\2\u0132\u0133\5\n\6\2\u0133\u0134\7\4\2\2\u0134\33\3\2\2\2\u0135"+ - "\u0136\t\t\2\2\u0136\35\3\2\2\2\u0137\u013c\5*\26\2\u0138\u013a\7\f\2"+ - "\2\u0139\u0138\3\2\2\2\u0139\u013a\3\2\2\2\u013a\u013b\3\2\2\2\u013b\u013d"+ - "\5B\"\2\u013c\u0139\3\2\2\2\u013c\u013d\3\2\2\2\u013d\37\3\2\2\2\u013e"+ - "\u0142\5(\25\2\u013f\u0141\5\"\22\2\u0140\u013f\3\2\2\2\u0141\u0144\3"+ - "\2\2\2\u0142\u0140\3\2\2\2\u0142\u0143\3\2\2\2\u0143!\3\2\2\2\u0144\u0142"+ - "\3\2\2\2\u0145\u0146\5$\23\2\u0146\u0147\7(\2\2\u0147\u0149\5(\25\2\u0148"+ - "\u014a\5&\24\2\u0149\u0148\3\2\2\2\u0149\u014a\3\2\2\2\u014a\u0151\3\2"+ - "\2\2\u014b\u014c\7.\2\2\u014c\u014d\5$\23\2\u014d\u014e\7(\2\2\u014e\u014f"+ - "\5(\25\2\u014f\u0151\3\2\2\2\u0150\u0145\3\2\2\2\u0150\u014b\3\2\2\2\u0151"+ - "#\3\2\2\2\u0152\u0154\7&\2\2\u0153\u0152\3\2\2\2\u0153\u0154\3\2\2\2\u0154"+ - "\u0162\3\2\2\2\u0155\u0157\7)\2\2\u0156\u0158\7\65\2\2\u0157\u0156\3\2"+ - "\2\2\u0157\u0158\3\2\2\2\u0158\u0162\3\2\2\2\u0159\u015b\79\2\2\u015a"+ - "\u015c\7\65\2\2\u015b\u015a\3\2\2\2\u015b\u015c\3\2\2\2\u015c\u0162\3"+ - "\2\2\2\u015d\u015f\7 \2\2\u015e\u0160\7\65\2\2\u015f\u015e\3\2\2\2\u015f"+ - "\u0160\3\2\2\2\u0160\u0162\3\2\2\2\u0161\u0153\3\2\2\2\u0161\u0155\3\2"+ - "\2\2\u0161\u0159\3\2\2\2\u0161\u015d\3\2\2\2\u0162%\3\2\2\2\u0163\u0164"+ - "\7\61\2\2\u0164\u0172\5,\27\2\u0165\u0166\7F\2\2\u0166\u0167\7\3\2\2\u0167"+ - "\u016c\5B\"\2\u0168\u0169\7\5\2\2\u0169\u016b\5B\"\2\u016a\u0168\3\2\2"+ - "\2\u016b\u016e\3\2\2\2\u016c\u016a\3\2\2\2\u016c\u016d\3\2\2\2\u016d\u016f"+ - "\3\2\2\2\u016e\u016c\3\2\2\2\u016f\u0170\7\4\2\2\u0170\u0172\3\2\2\2\u0171"+ - "\u0163\3\2\2\2\u0171\u0165\3\2\2\2\u0172\'\3\2\2\2\u0173\u0178\5D#\2\u0174"+ - "\u0176\7\f\2\2\u0175\u0174\3\2\2\2\u0175\u0176\3\2\2\2\u0176\u0177\3\2"+ - "\2\2\u0177\u0179\5@!\2\u0178\u0175\3\2\2\2\u0178\u0179\3\2\2\2\u0179\u018d"+ - "\3\2\2\2\u017a\u017b\7\3\2\2\u017b\u017c\5\n\6\2\u017c\u0181\7\4\2\2\u017d"+ - "\u017f\7\f\2\2\u017e\u017d\3\2\2\2\u017e\u017f\3\2\2\2\u017f\u0180\3\2"+ - "\2\2\u0180\u0182\5@!\2\u0181\u017e\3\2\2\2\u0181\u0182\3\2\2\2\u0182\u018d"+ - "\3\2\2\2\u0183\u0184\7\3\2\2\u0184\u0185\5 \21\2\u0185\u018a\7\4\2\2\u0186"+ - "\u0188\7\f\2\2\u0187\u0186\3\2\2\2\u0187\u0188\3\2\2\2\u0188\u0189\3\2"+ - "\2\2\u0189\u018b\5@!\2\u018a\u0187\3\2\2\2\u018a\u018b\3\2\2\2\u018b\u018d"+ - "\3\2\2\2\u018c\u0173\3\2\2\2\u018c\u017a\3\2\2\2\u018c\u0183\3\2\2\2\u018d"+ - ")\3\2\2\2\u018e\u018f\5,\27\2\u018f+\3\2\2\2\u0190\u0191\b\27\1\2\u0191"+ - "\u0192\7/\2\2\u0192\u01c2\5,\27\n\u0193\u0194\7\32\2\2\u0194\u0195\7\3"+ - "\2\2\u0195\u0196\5\b\5\2\u0196\u0197\7\4\2\2\u0197\u01c2\3\2\2\2\u0198"+ - "\u0199\7;\2\2\u0199\u019a\7\3\2\2\u019a\u019f\5L\'\2\u019b\u019c\7\5\2"+ - "\2\u019c\u019e\5L\'\2\u019d\u019b\3\2\2\2\u019e\u01a1\3\2\2\2\u019f\u019d"+ - "\3\2\2\2\u019f\u01a0\3\2\2\2\u01a0\u01a2\3\2\2\2\u01a1\u019f\3\2\2\2\u01a2"+ - "\u01a3\7\4\2\2\u01a3\u01c2\3\2\2\2\u01a4\u01a5\7-\2\2\u01a5\u01a6\7\3"+ - "\2\2\u01a6\u01a7\5@!\2\u01a7\u01a8\7\5\2\2\u01a8\u01ad\5L\'\2\u01a9\u01aa"+ - "\7\5\2\2\u01aa\u01ac\5L\'\2\u01ab\u01a9\3\2\2\2\u01ac\u01af\3\2\2\2\u01ad"+ - "\u01ab\3\2\2\2\u01ad\u01ae\3\2\2\2\u01ae\u01b0\3\2\2\2\u01af\u01ad\3\2"+ - "\2\2\u01b0\u01b1\7\4\2\2\u01b1\u01c2\3\2\2\2\u01b2\u01b3\7-\2\2\u01b3"+ - "\u01b4\7\3\2\2\u01b4\u01b5\5L\'\2\u01b5\u01b6\7\5\2\2\u01b6\u01bb\5L\'"+ - "\2\u01b7\u01b8\7\5\2\2\u01b8\u01ba\5L\'\2\u01b9\u01b7\3\2\2\2\u01ba\u01bd"+ - "\3\2\2\2\u01bb\u01b9\3\2\2\2\u01bb\u01bc\3\2\2\2\u01bc\u01be\3\2\2\2\u01bd"+ - "\u01bb\3\2\2\2\u01be\u01bf\7\4\2\2\u01bf\u01c2\3\2\2\2\u01c0\u01c2\5."+ - "\30\2\u01c1\u0190\3\2\2\2\u01c1\u0193\3\2\2\2\u01c1\u0198\3\2\2\2\u01c1"+ - "\u01a4\3\2\2\2\u01c1\u01b2\3\2\2\2\u01c1\u01c0\3\2\2\2\u01c2\u01cb\3\2"+ - "\2\2\u01c3\u01c4\f\4\2\2\u01c4\u01c5\7\n\2\2\u01c5\u01ca\5,\27\5\u01c6"+ - "\u01c7\f\3\2\2\u01c7\u01c8\7\63\2\2\u01c8\u01ca\5,\27\4\u01c9\u01c3\3"+ - "\2\2\2\u01c9\u01c6\3\2\2\2\u01ca\u01cd\3\2\2\2\u01cb\u01c9\3\2\2\2\u01cb"+ - "\u01cc\3\2\2\2\u01cc-\3\2\2\2\u01cd\u01cb\3\2\2\2\u01ce\u01d0\5\64\33"+ - "\2\u01cf\u01d1\5\60\31\2\u01d0\u01cf\3\2\2\2\u01d0\u01d1\3\2\2\2\u01d1"+ - "/\3\2\2\2\u01d2\u01d4\7/\2\2\u01d3\u01d2\3\2\2\2\u01d3\u01d4\3\2\2\2\u01d4"+ - "\u01d5\3\2\2\2\u01d5\u01d6\7\16\2\2\u01d6\u01d7\5\64\33\2\u01d7\u01d8"+ - "\7\n\2\2\u01d8\u01d9\5\64\33\2\u01d9\u0201\3\2\2\2\u01da\u01dc\7/\2\2"+ - "\u01db\u01da\3\2\2\2\u01db\u01dc\3\2\2\2\u01dc\u01dd\3\2\2\2\u01dd\u01de"+ - "\7%\2\2\u01de\u01df\7\3\2\2\u01df\u01e4\5*\26\2\u01e0\u01e1\7\5\2\2\u01e1"+ - "\u01e3\5*\26\2\u01e2\u01e0\3\2\2\2\u01e3\u01e6\3\2\2\2\u01e4\u01e2\3\2"+ - "\2\2\u01e4\u01e5\3\2\2\2\u01e5\u01e7\3\2\2\2\u01e6\u01e4\3\2\2\2\u01e7"+ - "\u01e8\7\4\2\2\u01e8\u0201\3\2\2\2\u01e9\u01eb\7/\2\2\u01ea\u01e9\3\2"+ - "\2\2\u01ea\u01eb\3\2\2\2\u01eb\u01ec\3\2\2\2\u01ec\u01ed\7%\2\2\u01ed"+ - "\u01ee\7\3\2\2\u01ee\u01ef\5\b\5\2\u01ef\u01f0\7\4\2\2\u01f0\u0201\3\2"+ - "\2\2\u01f1\u01f3\7/\2\2\u01f2\u01f1\3\2\2\2\u01f2\u01f3\3\2\2\2\u01f3"+ - "\u01f4\3\2\2\2\u01f4\u01f5\7*\2\2\u01f5\u0201\5\62\32\2\u01f6\u01f8\7"+ - "/\2\2\u01f7\u01f6\3\2\2\2\u01f7\u01f8\3\2\2\2\u01f8\u01f9\3\2\2\2\u01f9"+ - "\u01fa\7:\2\2\u01fa\u0201\5L\'\2\u01fb\u01fd\7\'\2\2\u01fc\u01fe\7/\2"+ - "\2\u01fd\u01fc\3\2\2\2\u01fd\u01fe\3\2\2\2\u01fe\u01ff\3\2\2\2\u01ff\u0201"+ - "\7\60\2\2\u0200\u01d3\3\2\2\2\u0200\u01db\3\2\2\2\u0200\u01ea\3\2\2\2"+ - "\u0200\u01f2\3\2\2\2\u0200\u01f7\3\2\2\2\u0200\u01fb\3\2\2\2\u0201\61"+ - "\3\2\2\2\u0202\u0205\5L\'\2\u0203\u0204\7\30\2\2\u0204\u0206\5L\'\2\u0205"+ - "\u0203\3\2\2\2\u0205\u0206\3\2\2\2\u0206\63\3\2\2\2\u0207\u0208\b\33\1"+ - "\2\u0208\u020c\5\66\34\2\u0209\u020a\t\n\2\2\u020a\u020c\5\64\33\6\u020b"+ - "\u0207\3\2\2\2\u020b\u0209\3\2\2\2\u020c\u0219\3\2\2\2\u020d\u020e\f\5"+ - "\2\2\u020e\u020f\t\13\2\2\u020f\u0218\5\64\33\6\u0210\u0211\f\4\2\2\u0211"+ - "\u0212\t\n\2\2\u0212\u0218\5\64\33\5\u0213\u0214\f\3\2\2\u0214\u0215\5"+ - ":\36\2\u0215\u0216\5\64\33\4\u0216\u0218\3\2\2\2\u0217\u020d\3\2\2\2\u0217"+ - "\u0210\3\2\2\2\u0217\u0213\3\2\2\2\u0218\u021b\3\2\2\2\u0219\u0217\3\2"+ - "\2\2\u0219\u021a\3\2\2\2\u021a\65\3\2\2\2\u021b\u0219\3\2\2\2\u021c\u021d"+ - "\7\20\2\2\u021d\u021e\7\3\2\2\u021e\u021f\5*\26\2\u021f\u0220\7\f\2\2"+ - "\u0220\u0221\5> \2\u0221\u0222\7\4\2\2\u0222\u024e\3\2\2\2\u0223\u0224"+ - "\7\34\2\2\u0224\u0225\7\3\2\2\u0225\u0226\5B\"\2\u0226\u0227\7\37\2\2"+ - "\u0227\u0228\5\64\33\2\u0228\u0229\7\4\2\2\u0229\u024e\3\2\2\2\u022a\u024e"+ - "\58\35\2\u022b\u024e\7R\2\2\u022c\u022d\5@!\2\u022d\u022e\7V\2\2\u022e"+ - "\u0230\3\2\2\2\u022f\u022c\3\2\2\2\u022f\u0230\3\2\2\2\u0230\u0231\3\2"+ - "\2\2\u0231\u024e\7R\2\2\u0232\u0233\5B\"\2\u0233\u023f\7\3\2\2\u0234\u0236"+ - "\5\34\17\2\u0235\u0234\3\2\2\2\u0235\u0236\3\2\2\2\u0236\u0237\3\2\2\2"+ - "\u0237\u023c\5*\26\2\u0238\u0239\7\5\2\2\u0239\u023b\5*\26\2\u023a\u0238"+ - "\3\2\2\2\u023b\u023e\3\2\2\2\u023c\u023a\3\2\2\2\u023c\u023d\3\2\2\2\u023d"+ - "\u0240\3\2\2\2\u023e\u023c\3\2\2\2\u023f\u0235\3\2\2\2\u023f\u0240\3\2"+ - "\2\2\u0240\u0241\3\2\2\2\u0241\u0242\7\4\2\2\u0242\u024e\3\2\2\2\u0243"+ - "\u0244\7\3\2\2\u0244\u0245\5\b\5\2\u0245\u0246\7\4\2\2\u0246\u024e\3\2"+ - "\2\2\u0247\u024e\5B\"\2\u0248\u024e\5@!\2\u0249\u024a\7\3\2\2\u024a\u024b"+ - "\5*\26\2\u024b\u024c\7\4\2\2\u024c\u024e\3\2\2\2\u024d\u021c\3\2\2\2\u024d"+ - "\u0223\3\2\2\2\u024d\u022a\3\2\2\2\u024d\u022b\3\2\2\2\u024d\u022f\3\2"+ - "\2\2\u024d\u0232\3\2\2\2\u024d\u0243\3\2\2\2\u024d\u0247\3\2\2\2\u024d"+ - "\u0248\3\2\2\2\u024d\u0249\3\2\2\2\u024e\67\3\2\2\2\u024f\u0259\7\60\2"+ - "\2\u0250\u0259\5J&\2\u0251\u0259\5<\37\2\u0252\u0254\7X\2\2\u0253\u0252"+ - "\3\2\2\2\u0254\u0255\3\2\2\2\u0255\u0253\3\2\2\2\u0255\u0256\3\2\2\2\u0256"+ - "\u0259\3\2\2\2\u0257\u0259\7W\2\2\u0258\u024f\3\2\2\2\u0258\u0250\3\2"+ - "\2\2\u0258\u0251\3\2\2\2\u0258\u0253\3\2\2\2\u0258\u0257\3\2\2\2\u0259"+ - "9\3\2\2\2\u025a\u025b\t\f\2\2\u025b;\3\2\2\2\u025c\u025d\t\r\2\2\u025d"+ - "=\3\2\2\2\u025e\u025f\5B\"\2\u025f?\3\2\2\2\u0260\u0261\5B\"\2\u0261\u0262"+ - "\7V\2\2\u0262\u0264\3\2\2\2\u0263\u0260\3\2\2\2\u0264\u0267\3\2\2\2\u0265"+ - "\u0263\3\2\2\2\u0265\u0266\3\2\2\2\u0266\u0268\3\2\2\2\u0267\u0265\3\2"+ - "\2\2\u0268\u0269\5B\"\2\u0269A\3\2\2\2\u026a\u026d\5F$\2\u026b\u026d\5"+ - "H%\2\u026c\u026a\3\2\2\2\u026c\u026b\3\2\2\2\u026dC\3\2\2\2\u026e\u026f"+ - "\5B\"\2\u026f\u0270\7\6\2\2\u0270\u0272\3\2\2\2\u0271\u026e\3\2\2\2\u0271"+ - "\u0272\3\2\2\2\u0272\u0273\3\2\2\2\u0273\u027b\7]\2\2\u0274\u0275\5B\""+ - "\2\u0275\u0276\7\6\2\2\u0276\u0278\3\2\2\2\u0277\u0274\3\2\2\2\u0277\u0278"+ - "\3\2\2\2\u0278\u0279\3\2\2\2\u0279\u027b\5B\"\2\u027a\u0271\3\2\2\2\u027a"+ - "\u0277\3\2\2\2\u027bE\3\2\2\2\u027c\u027f\7^\2\2\u027d\u027f\7_\2\2\u027e"+ - "\u027c\3\2\2\2\u027e\u027d\3\2\2\2\u027fG\3\2\2\2\u0280\u0284\7[\2\2\u0281"+ - "\u0284\5N(\2\u0282\u0284\7\\\2\2\u0283\u0280\3\2\2\2\u0283\u0281\3\2\2"+ - "\2\u0283\u0282\3\2\2\2\u0284I\3\2\2\2\u0285\u0288\7Z\2\2\u0286\u0288\7"+ - "Y\2\2\u0287\u0285\3\2\2\2\u0287\u0286\3\2\2\2\u0288K\3\2\2\2\u0289\u028a"+ - "\t\16\2\2\u028aM\3\2\2\2\u028b\u028c\t\17\2\2\u028cO\3\2\2\2`_aenptz}"+ - "\u0088\u008b\u0095\u0098\u009b\u009e\u00a6\u00a9\u00af\u00b3\u00b6\u00b9"+ - "\u00bc\u00c3\u00cb\u00ce\u00da\u00dd\u00e1\u00e8\u00ec\u00f0\u00f7\u00fb"+ - "\u00ff\u0104\u0108\u0110\u0114\u011b\u0126\u0129\u012d\u0139\u013c\u0142"+ - "\u0149\u0150\u0153\u0157\u015b\u015f\u0161\u016c\u0171\u0175\u0178\u017e"+ - "\u0181\u0187\u018a\u018c\u019f\u01ad\u01bb\u01c1\u01c9\u01cb\u01d0\u01d3"+ - "\u01db\u01e4\u01ea\u01f2\u01f7\u01fd\u0200\u0205\u020b\u0217\u0219\u022f"+ - "\u0235\u023c\u023f\u024d\u0255\u0258\u0265\u026c\u0271\u0277\u027a\u027e"+ - "\u0283\u0287"; + "\t!\4\"\t\"\4#\t#\4$\t$\4%\t%\4&\t&\4\'\t\'\4(\t(\4)\t)\4*\t*\4+\t+\4"+ + ",\t,\4-\t-\4.\t.\4/\t/\4\60\t\60\3\2\3\2\3\2\3\3\3\3\3\3\3\4\3\4\3\4\3"+ + "\4\3\4\3\4\3\4\3\4\3\4\7\4p\n\4\f\4\16\4s\13\4\3\4\5\4v\n\4\3\4\3\4\3"+ + "\4\3\4\3\4\3\4\3\4\7\4\177\n\4\f\4\16\4\u0082\13\4\3\4\5\4\u0085\n\4\3"+ + "\4\3\4\3\4\3\4\5\4\u008b\n\4\3\4\5\4\u008e\n\4\3\4\3\4\3\4\3\4\3\4\3\4"+ + "\3\4\3\4\3\4\5\4\u0099\n\4\3\4\5\4\u009c\n\4\3\4\3\4\3\4\3\4\3\4\3\4\3"+ + "\4\3\4\5\4\u00a6\n\4\3\4\5\4\u00a9\n\4\3\4\5\4\u00ac\n\4\3\4\5\4\u00af"+ + "\n\4\3\4\3\4\3\4\3\4\7\4\u00b5\n\4\f\4\16\4\u00b8\13\4\5\4\u00ba\n\4\3"+ + "\4\3\4\3\4\3\4\5\4\u00c0\n\4\3\4\3\4\5\4\u00c4\n\4\3\4\5\4\u00c7\n\4\3"+ + "\4\5\4\u00ca\n\4\3\4\5\4\u00cd\n\4\3\4\3\4\3\4\3\4\3\4\5\4\u00d4\n\4\3"+ + "\5\3\5\3\5\3\5\7\5\u00da\n\5\f\5\16\5\u00dd\13\5\5\5\u00df\n\5\3\5\3\5"+ + "\3\6\3\6\3\6\3\6\3\6\3\6\7\6\u00e9\n\6\f\6\16\6\u00ec\13\6\5\6\u00ee\n"+ + "\6\3\6\5\6\u00f1\n\6\3\7\3\7\3\7\3\7\3\7\5\7\u00f8\n\7\3\b\3\b\3\b\3\b"+ + "\3\b\5\b\u00ff\n\b\3\t\3\t\5\t\u0103\n\t\3\n\3\n\5\n\u0107\n\n\3\n\3\n"+ + "\3\n\7\n\u010c\n\n\f\n\16\n\u010f\13\n\3\n\5\n\u0112\n\n\3\n\3\n\5\n\u0116"+ + "\n\n\3\n\3\n\3\n\5\n\u011b\n\n\3\n\3\n\5\n\u011f\n\n\3\13\3\13\3\13\3"+ + "\13\7\13\u0125\n\13\f\13\16\13\u0128\13\13\3\f\5\f\u012b\n\f\3\f\3\f\3"+ + "\f\7\f\u0130\n\f\f\f\16\f\u0133\13\f\3\r\3\r\3\16\3\16\3\16\3\16\7\16"+ + "\u013b\n\16\f\16\16\16\u013e\13\16\5\16\u0140\n\16\3\16\3\16\5\16\u0144"+ + "\n\16\3\17\3\17\3\17\3\17\3\17\3\17\3\20\3\20\3\21\3\21\5\21\u0150\n\21"+ + "\3\21\5\21\u0153\n\21\3\22\3\22\7\22\u0157\n\22\f\22\16\22\u015a\13\22"+ + "\3\23\3\23\3\23\3\23\5\23\u0160\n\23\3\23\3\23\3\23\3\23\3\23\5\23\u0167"+ + "\n\23\3\24\5\24\u016a\n\24\3\24\3\24\5\24\u016e\n\24\3\24\3\24\5\24\u0172"+ + "\n\24\3\24\3\24\5\24\u0176\n\24\5\24\u0178\n\24\3\25\3\25\3\25\3\25\3"+ + "\25\3\25\3\25\7\25\u0181\n\25\f\25\16\25\u0184\13\25\3\25\3\25\5\25\u0188"+ + "\n\25\3\26\3\26\5\26\u018c\n\26\3\26\5\26\u018f\n\26\3\26\3\26\3\26\3"+ + "\26\5\26\u0195\n\26\3\26\5\26\u0198\n\26\3\26\3\26\3\26\3\26\5\26\u019e"+ + "\n\26\3\26\5\26\u01a1\n\26\5\26\u01a3\n\26\3\27\3\27\3\30\3\30\3\30\3"+ + "\30\3\30\3\30\3\30\3\30\3\30\3\30\3\30\3\30\3\30\7\30\u01b4\n\30\f\30"+ + "\16\30\u01b7\13\30\3\30\3\30\3\30\3\30\3\30\3\30\3\30\3\30\3\30\7\30\u01c2"+ + "\n\30\f\30\16\30\u01c5\13\30\3\30\3\30\3\30\3\30\3\30\3\30\3\30\3\30\3"+ + "\30\7\30\u01d0\n\30\f\30\16\30\u01d3\13\30\3\30\3\30\3\30\5\30\u01d8\n"+ + "\30\3\30\3\30\3\30\3\30\3\30\3\30\7\30\u01e0\n\30\f\30\16\30\u01e3\13"+ + "\30\3\31\3\31\5\31\u01e7\n\31\3\32\5\32\u01ea\n\32\3\32\3\32\3\32\3\32"+ + "\3\32\3\32\5\32\u01f2\n\32\3\32\3\32\3\32\3\32\3\32\7\32\u01f9\n\32\f"+ + "\32\16\32\u01fc\13\32\3\32\3\32\3\32\5\32\u0201\n\32\3\32\3\32\3\32\3"+ + "\32\3\32\3\32\5\32\u0209\n\32\3\32\3\32\3\32\5\32\u020e\n\32\3\32\3\32"+ + "\3\32\3\32\5\32\u0214\n\32\3\32\5\32\u0217\n\32\3\33\3\33\5\33\u021b\n"+ + "\33\3\34\3\34\3\34\3\34\3\34\3\34\5\34\u0223\n\34\3\35\3\35\3\35\3\35"+ + "\5\35\u0229\n\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35\3\35\7\35"+ + "\u0235\n\35\f\35\16\35\u0238\13\35\3\36\3\36\3\36\3\36\3\36\3\36\3\36"+ + "\5\36\u0241\n\36\3\36\3\36\3\36\3\36\3\36\3\36\3\36\3\36\3\36\3\36\3\36"+ + "\3\36\5\36\u024f\n\36\3\37\3\37\3\37\3\37\3\37\5\37\u0256\n\37\3 \3 \3"+ + " \3 \3 \3 \3 \3!\3!\3!\3!\3!\5!\u0264\n!\3\"\3\"\3\"\3\"\3\"\3\"\3\"\3"+ + "#\3#\3#\3#\3#\5#\u0272\n#\3$\3$\3$\5$\u0277\n$\3$\3$\3$\7$\u027c\n$\f"+ + "$\16$\u027f\13$\5$\u0281\n$\3$\3$\3%\3%\3%\3%\6%\u0289\n%\r%\16%\u028a"+ + "\3%\3%\3%\3%\3%\3%\3%\3%\3%\3%\3%\3%\3%\3%\3%\3%\3%\5%\u029e\n%\3&\3&"+ + "\3\'\3\'\3(\3(\3)\3)\3)\7)\u02a9\n)\f)\16)\u02ac\13)\3)\3)\3*\3*\5*\u02b2"+ + "\n*\3+\3+\3+\5+\u02b7\n+\3+\3+\3+\3+\5+\u02bd\n+\3+\5+\u02c0\n+\3,\3,"+ + "\5,\u02c4\n,\3-\3-\3-\5-\u02c9\n-\3.\3.\5.\u02cd\n.\3/\3/\3\60\3\60\3"+ + "\60\2\4.8\61\2\4\6\b\n\f\16\20\22\24\26\30\32\34\36 \"$&(*,.\60\62\64"+ + "\668:<>@BDFHJLNPRTVXZ\\^\2\20\b\2\7\7\t\t\31\31,,\62\62\66\66\4\2\"\""+ + "BB\4\2\t\t\62\62\4\2\37\37%%\3\2\25\26\4\2\7\7aa\4\2\r\r\25\25\4\2\7\7"+ + "\27\27\3\2XY\3\2Z\\\3\2RW\4\2\35\35CC\3\2_`\20\2\b\t\22\24\31\31\33\33"+ + "\36\36!\",,\62\62\668:<>?ABDEGG\u032e\2`\3\2\2\2\4c\3\2\2\2\6\u00d3\3"+ + "\2\2\2\b\u00de\3\2\2\2\n\u00e2\3\2\2\2\f\u00f7\3\2\2\2\16\u00fe\3\2\2"+ + "\2\20\u0100\3\2\2\2\22\u0104\3\2\2\2\24\u0120\3\2\2\2\26\u012a\3\2\2\2"+ + "\30\u0134\3\2\2\2\32\u0143\3\2\2\2\34\u0145\3\2\2\2\36\u014b\3\2\2\2 "+ + "\u014d\3\2\2\2\"\u0154\3\2\2\2$\u0166\3\2\2\2&\u0177\3\2\2\2(\u0187\3"+ + "\2\2\2*\u01a2\3\2\2\2,\u01a4\3\2\2\2.\u01d7\3\2\2\2\60\u01e4\3\2\2\2\62"+ + "\u0216\3\2\2\2\64\u0218\3\2\2\2\66\u0222\3\2\2\28\u0228\3\2\2\2:\u024e"+ + "\3\2\2\2<\u0255\3\2\2\2>\u0257\3\2\2\2@\u0263\3\2\2\2B\u0265\3\2\2\2D"+ + "\u0271\3\2\2\2F\u0273\3\2\2\2H\u029d\3\2\2\2J\u029f\3\2\2\2L\u02a1\3\2"+ + "\2\2N\u02a3\3\2\2\2P\u02aa\3\2\2\2R\u02b1\3\2\2\2T\u02bf\3\2\2\2V\u02c3"+ + "\3\2\2\2X\u02c8\3\2\2\2Z\u02cc\3\2\2\2\\\u02ce\3\2\2\2^\u02d0\3\2\2\2"+ + "`a\5\6\4\2ab\7\2\2\3b\3\3\2\2\2cd\5,\27\2de\7\2\2\3e\5\3\2\2\2f\u00d4"+ + "\5\b\5\2gu\7\33\2\2hq\7\3\2\2ij\78\2\2jp\t\2\2\2kl\7\36\2\2lp\t\3\2\2"+ + "mn\7G\2\2np\5L\'\2oi\3\2\2\2ok\3\2\2\2om\3\2\2\2ps\3\2\2\2qo\3\2\2\2q"+ + "r\3\2\2\2rt\3\2\2\2sq\3\2\2\2tv\7\4\2\2uh\3\2\2\2uv\3\2\2\2vw\3\2\2\2"+ + "w\u00d4\5\6\4\2x\u0084\7\24\2\2y\u0080\7\3\2\2z{\78\2\2{\177\t\4\2\2|"+ + "}\7\36\2\2}\177\t\3\2\2~z\3\2\2\2~|\3\2\2\2\177\u0082\3\2\2\2\u0080~\3"+ + "\2\2\2\u0080\u0081\3\2\2\2\u0081\u0083\3\2\2\2\u0082\u0080\3\2\2\2\u0083"+ + "\u0085\7\4\2\2\u0084y\3\2\2\2\u0084\u0085\3\2\2\2\u0085\u0086\3\2\2\2"+ + "\u0086\u00d4\5\6\4\2\u0087\u0088\7>\2\2\u0088\u008d\7A\2\2\u0089\u008b"+ + "\7*\2\2\u008a\u0089\3\2\2\2\u008a\u008b\3\2\2\2\u008b\u008c\3\2\2\2\u008c"+ + "\u008e\5\64\33\2\u008d\u008a\3\2\2\2\u008d\u008e\3\2\2\2\u008e\u00d4\3"+ + "\2\2\2\u008f\u0090\7>\2\2\u0090\u0091\7\23\2\2\u0091\u0092\t\5\2\2\u0092"+ + "\u00d4\5T+\2\u0093\u0094\t\6\2\2\u0094\u00d4\5T+\2\u0095\u0096\7>\2\2"+ + "\u0096\u009b\7!\2\2\u0097\u0099\7*\2\2\u0098\u0097\3\2\2\2\u0098\u0099"+ + "\3\2\2\2\u0099\u009a\3\2\2\2\u009a\u009c\5\64\33\2\u009b\u0098\3\2\2\2"+ + "\u009b\u009c\3\2\2\2\u009c\u00d4\3\2\2\2\u009d\u009e\7>\2\2\u009e\u00d4"+ + "\7<\2\2\u009f\u00a0\7?\2\2\u00a0\u00d4\7\22\2\2\u00a1\u00a2\7?\2\2\u00a2"+ + "\u00a8\7A\2\2\u00a3\u00a5\7\21\2\2\u00a4\u00a6\7*\2\2\u00a5\u00a4\3\2"+ + "\2\2\u00a5\u00a6\3\2\2\2\u00a6\u00a7\3\2\2\2\u00a7\u00a9\5\64\33\2\u00a8"+ + "\u00a3\3\2\2\2\u00a8\u00a9\3\2\2\2\u00a9\u00ae\3\2\2\2\u00aa\u00ac\7*"+ + "\2\2\u00ab\u00aa\3\2\2\2\u00ab\u00ac\3\2\2\2\u00ac\u00ad\3\2\2\2\u00ad"+ + "\u00af\5\64\33\2\u00ae\u00ab\3\2\2\2\u00ae\u00af\3\2\2\2\u00af\u00b9\3"+ + "\2\2\2\u00b0\u00b1\7D\2\2\u00b1\u00b6\5\\/\2\u00b2\u00b3\7\5\2\2\u00b3"+ + "\u00b5\5\\/\2\u00b4\u00b2\3\2\2\2\u00b5\u00b8\3\2\2\2\u00b6\u00b4\3\2"+ + "\2\2\u00b6\u00b7\3\2\2\2\u00b7\u00ba\3\2\2\2\u00b8\u00b6\3\2\2\2\u00b9"+ + "\u00b0\3\2\2\2\u00b9\u00ba\3\2\2\2\u00ba\u00d4\3\2\2\2\u00bb\u00bc\7?"+ + "\2\2\u00bc\u00bf\7\23\2\2\u00bd\u00be\7\21\2\2\u00be\u00c0\5\\/\2\u00bf"+ + "\u00bd\3\2\2\2\u00bf\u00c0\3\2\2\2\u00c0\u00c6\3\2\2\2\u00c1\u00c3\7@"+ + "\2\2\u00c2\u00c4\7*\2\2\u00c3\u00c2\3\2\2\2\u00c3\u00c4\3\2\2\2\u00c4"+ + "\u00c5\3\2\2\2\u00c5\u00c7\5\64\33\2\u00c6\u00c1\3\2\2\2\u00c6\u00c7\3"+ + "\2\2\2\u00c7\u00cc\3\2\2\2\u00c8\u00ca\7*\2\2\u00c9\u00c8\3\2\2\2\u00c9"+ + "\u00ca\3\2\2\2\u00ca\u00cb\3\2\2\2\u00cb\u00cd\5\64\33\2\u00cc\u00c9\3"+ + "\2\2\2\u00cc\u00cd\3\2\2\2\u00cd\u00d4\3\2\2\2\u00ce\u00cf\7?\2\2\u00cf"+ + "\u00d4\7E\2\2\u00d0\u00d1\7?\2\2\u00d1\u00d2\7@\2\2\u00d2\u00d4\7E\2\2"+ + "\u00d3f\3\2\2\2\u00d3g\3\2\2\2\u00d3x\3\2\2\2\u00d3\u0087\3\2\2\2\u00d3"+ + "\u008f\3\2\2\2\u00d3\u0093\3\2\2\2\u00d3\u0095\3\2\2\2\u00d3\u009d\3\2"+ + "\2\2\u00d3\u009f\3\2\2\2\u00d3\u00a1\3\2\2\2\u00d3\u00bb\3\2\2\2\u00d3"+ + "\u00ce\3\2\2\2\u00d3\u00d0\3\2\2\2\u00d4\7\3\2\2\2\u00d5\u00d6\7I\2\2"+ + "\u00d6\u00db\5\34\17\2\u00d7\u00d8\7\5\2\2\u00d8\u00da\5\34\17\2\u00d9"+ + "\u00d7\3\2\2\2\u00da\u00dd\3\2\2\2\u00db\u00d9\3\2\2\2\u00db\u00dc\3\2"+ + "\2\2\u00dc\u00df\3\2\2\2\u00dd\u00db\3\2\2\2\u00de\u00d5\3\2\2\2\u00de"+ + "\u00df\3\2\2\2\u00df\u00e0\3\2\2\2\u00e0\u00e1\5\n\6\2\u00e1\t\3\2\2\2"+ + "\u00e2\u00ed\5\16\b\2\u00e3\u00e4\7\64\2\2\u00e4\u00e5\7\17\2\2\u00e5"+ + "\u00ea\5\20\t\2\u00e6\u00e7\7\5\2\2\u00e7\u00e9\5\20\t\2\u00e8\u00e6\3"+ + "\2\2\2\u00e9\u00ec\3\2\2\2\u00ea\u00e8\3\2\2\2\u00ea\u00eb\3\2\2\2\u00eb"+ + "\u00ee\3\2\2\2\u00ec\u00ea\3\2\2\2\u00ed\u00e3\3\2\2\2\u00ed\u00ee\3\2"+ + "\2\2\u00ee\u00f0\3\2\2\2\u00ef\u00f1\5\f\7\2\u00f0\u00ef\3\2\2\2\u00f0"+ + "\u00f1\3\2\2\2\u00f1\13\3\2\2\2\u00f2\u00f3\7+\2\2\u00f3\u00f8\t\7\2\2"+ + "\u00f4\u00f5\7L\2\2\u00f5\u00f6\t\7\2\2\u00f6\u00f8\7Q\2\2\u00f7\u00f2"+ + "\3\2\2\2\u00f7\u00f4\3\2\2\2\u00f8\r\3\2\2\2\u00f9\u00ff\5\22\n\2\u00fa"+ + "\u00fb\7\3\2\2\u00fb\u00fc\5\n\6\2\u00fc\u00fd\7\4\2\2\u00fd\u00ff\3\2"+ + "\2\2\u00fe\u00f9\3\2\2\2\u00fe\u00fa\3\2\2\2\u00ff\17\3\2\2\2\u0100\u0102"+ + "\5,\27\2\u0101\u0103\t\b\2\2\u0102\u0101\3\2\2\2\u0102\u0103\3\2\2\2\u0103"+ + "\21\3\2\2\2\u0104\u0106\7=\2\2\u0105\u0107\5\36\20\2\u0106\u0105\3\2\2"+ + "\2\u0106\u0107\3\2\2\2\u0107\u0108\3\2\2\2\u0108\u010d\5 \21\2\u0109\u010a"+ + "\7\5\2\2\u010a\u010c\5 \21\2\u010b\u0109\3\2\2\2\u010c\u010f\3\2\2\2\u010d"+ + "\u010b\3\2\2\2\u010d\u010e\3\2\2\2\u010e\u0111\3\2\2\2\u010f\u010d\3\2"+ + "\2\2\u0110\u0112\5\24\13\2\u0111\u0110\3\2\2\2\u0111\u0112\3\2\2\2\u0112"+ + "\u0115\3\2\2\2\u0113\u0114\7H\2\2\u0114\u0116\5.\30\2\u0115\u0113\3\2"+ + "\2\2\u0115\u0116\3\2\2\2\u0116\u011a\3\2\2\2\u0117\u0118\7#\2\2\u0118"+ + "\u0119\7\17\2\2\u0119\u011b\5\26\f\2\u011a\u0117\3\2\2\2\u011a\u011b\3"+ + "\2\2\2\u011b\u011e\3\2\2\2\u011c\u011d\7$\2\2\u011d\u011f\5.\30\2\u011e"+ + "\u011c\3\2\2\2\u011e\u011f\3\2\2\2\u011f\23\3\2\2\2\u0120\u0121\7\37\2"+ + "\2\u0121\u0126\5\"\22\2\u0122\u0123\7\5\2\2\u0123\u0125\5\"\22\2\u0124"+ + "\u0122\3\2\2\2\u0125\u0128\3\2\2\2\u0126\u0124\3\2\2\2\u0126\u0127\3\2"+ + "\2\2\u0127\25\3\2\2\2\u0128\u0126\3\2\2\2\u0129\u012b\5\36\20\2\u012a"+ + "\u0129\3\2\2\2\u012a\u012b\3\2\2\2\u012b\u012c\3\2\2\2\u012c\u0131\5\30"+ + "\r\2\u012d\u012e\7\5\2\2\u012e\u0130\5\30\r\2\u012f\u012d\3\2\2\2\u0130"+ + "\u0133\3\2\2\2\u0131\u012f\3\2\2\2\u0131\u0132\3\2\2\2\u0132\27\3\2\2"+ + "\2\u0133\u0131\3\2\2\2\u0134\u0135\5\32\16\2\u0135\31\3\2\2\2\u0136\u013f"+ + "\7\3\2\2\u0137\u013c\5,\27\2\u0138\u0139\7\5\2\2\u0139\u013b\5,\27\2\u013a"+ + "\u0138\3\2\2\2\u013b\u013e\3\2\2\2\u013c\u013a\3\2\2\2\u013c\u013d\3\2"+ + "\2\2\u013d\u0140\3\2\2\2\u013e\u013c\3\2\2\2\u013f\u0137\3\2\2\2\u013f"+ + "\u0140\3\2\2\2\u0140\u0141\3\2\2\2\u0141\u0144\7\4\2\2\u0142\u0144\5,"+ + "\27\2\u0143\u0136\3\2\2\2\u0143\u0142\3\2\2\2\u0144\33\3\2\2\2\u0145\u0146"+ + "\5R*\2\u0146\u0147\7\f\2\2\u0147\u0148\7\3\2\2\u0148\u0149\5\n\6\2\u0149"+ + "\u014a\7\4\2\2\u014a\35\3\2\2\2\u014b\u014c\t\t\2\2\u014c\37\3\2\2\2\u014d"+ + "\u0152\5,\27\2\u014e\u0150\7\f\2\2\u014f\u014e\3\2\2\2\u014f\u0150\3\2"+ + "\2\2\u0150\u0151\3\2\2\2\u0151\u0153\5R*\2\u0152\u014f\3\2\2\2\u0152\u0153"+ + "\3\2\2\2\u0153!\3\2\2\2\u0154\u0158\5*\26\2\u0155\u0157\5$\23\2\u0156"+ + "\u0155\3\2\2\2\u0157\u015a\3\2\2\2\u0158\u0156\3\2\2\2\u0158\u0159\3\2"+ + "\2\2\u0159#\3\2\2\2\u015a\u0158\3\2\2\2\u015b\u015c\5&\24\2\u015c\u015d"+ + "\7(\2\2\u015d\u015f\5*\26\2\u015e\u0160\5(\25\2\u015f\u015e\3\2\2\2\u015f"+ + "\u0160\3\2\2\2\u0160\u0167\3\2\2\2\u0161\u0162\7.\2\2\u0162\u0163\5&\24"+ + "\2\u0163\u0164\7(\2\2\u0164\u0165\5*\26\2\u0165\u0167\3\2\2\2\u0166\u015b"+ + "\3\2\2\2\u0166\u0161\3\2\2\2\u0167%\3\2\2\2\u0168\u016a\7&\2\2\u0169\u0168"+ + "\3\2\2\2\u0169\u016a\3\2\2\2\u016a\u0178\3\2\2\2\u016b\u016d\7)\2\2\u016c"+ + "\u016e\7\65\2\2\u016d\u016c\3\2\2\2\u016d\u016e\3\2\2\2\u016e\u0178\3"+ + "\2\2\2\u016f\u0171\79\2\2\u0170\u0172\7\65\2\2\u0171\u0170\3\2\2\2\u0171"+ + "\u0172\3\2\2\2\u0172\u0178\3\2\2\2\u0173\u0175\7 \2\2\u0174\u0176\7\65"+ + "\2\2\u0175\u0174\3\2\2\2\u0175\u0176\3\2\2\2\u0176\u0178\3\2\2\2\u0177"+ + "\u0169\3\2\2\2\u0177\u016b\3\2\2\2\u0177\u016f\3\2\2\2\u0177\u0173\3\2"+ + "\2\2\u0178\'\3\2\2\2\u0179\u017a\7\61\2\2\u017a\u0188\5.\30\2\u017b\u017c"+ + "\7F\2\2\u017c\u017d\7\3\2\2\u017d\u0182\5R*\2\u017e\u017f\7\5\2\2\u017f"+ + "\u0181\5R*\2\u0180\u017e\3\2\2\2\u0181\u0184\3\2\2\2\u0182\u0180\3\2\2"+ + "\2\u0182\u0183\3\2\2\2\u0183\u0185\3\2\2\2\u0184\u0182\3\2\2\2\u0185\u0186"+ + "\7\4\2\2\u0186\u0188\3\2\2\2\u0187\u0179\3\2\2\2\u0187\u017b\3\2\2\2\u0188"+ + ")\3\2\2\2\u0189\u018e\5T+\2\u018a\u018c\7\f\2\2\u018b\u018a\3\2\2\2\u018b"+ + "\u018c\3\2\2\2\u018c\u018d\3\2\2\2\u018d\u018f\5P)\2\u018e\u018b\3\2\2"+ + "\2\u018e\u018f\3\2\2\2\u018f\u01a3\3\2\2\2\u0190\u0191\7\3\2\2\u0191\u0192"+ + "\5\n\6\2\u0192\u0197\7\4\2\2\u0193\u0195\7\f\2\2\u0194\u0193\3\2\2\2\u0194"+ + "\u0195\3\2\2\2\u0195\u0196\3\2\2\2\u0196\u0198\5P)\2\u0197\u0194\3\2\2"+ + "\2\u0197\u0198\3\2\2\2\u0198\u01a3\3\2\2\2\u0199\u019a\7\3\2\2\u019a\u019b"+ + "\5\"\22\2\u019b\u01a0\7\4\2\2\u019c\u019e\7\f\2\2\u019d\u019c\3\2\2\2"+ + "\u019d\u019e\3\2\2\2\u019e\u019f\3\2\2\2\u019f\u01a1\5P)\2\u01a0\u019d"+ + "\3\2\2\2\u01a0\u01a1\3\2\2\2\u01a1\u01a3\3\2\2\2\u01a2\u0189\3\2\2\2\u01a2"+ + "\u0190\3\2\2\2\u01a2\u0199\3\2\2\2\u01a3+\3\2\2\2\u01a4\u01a5\5.\30\2"+ + "\u01a5-\3\2\2\2\u01a6\u01a7\b\30\1\2\u01a7\u01a8\7/\2\2\u01a8\u01d8\5"+ + ".\30\n\u01a9\u01aa\7\32\2\2\u01aa\u01ab\7\3\2\2\u01ab\u01ac\5\b\5\2\u01ac"+ + "\u01ad\7\4\2\2\u01ad\u01d8\3\2\2\2\u01ae\u01af\7;\2\2\u01af\u01b0\7\3"+ + "\2\2\u01b0\u01b5\5\\/\2\u01b1\u01b2\7\5\2\2\u01b2\u01b4\5\\/\2\u01b3\u01b1"+ + "\3\2\2\2\u01b4\u01b7\3\2\2\2\u01b5\u01b3\3\2\2\2\u01b5\u01b6\3\2\2\2\u01b6"+ + "\u01b8\3\2\2\2\u01b7\u01b5\3\2\2\2\u01b8\u01b9\7\4\2\2\u01b9\u01d8\3\2"+ + "\2\2\u01ba\u01bb\7-\2\2\u01bb\u01bc\7\3\2\2\u01bc\u01bd\5P)\2\u01bd\u01be"+ + "\7\5\2\2\u01be\u01c3\5\\/\2\u01bf\u01c0\7\5\2\2\u01c0\u01c2\5\\/\2\u01c1"+ + "\u01bf\3\2\2\2\u01c2\u01c5\3\2\2\2\u01c3\u01c1\3\2\2\2\u01c3\u01c4\3\2"+ + "\2\2\u01c4\u01c6\3\2\2\2\u01c5\u01c3\3\2\2\2\u01c6\u01c7\7\4\2\2\u01c7"+ + "\u01d8\3\2\2\2\u01c8\u01c9\7-\2\2\u01c9\u01ca\7\3\2\2\u01ca\u01cb\5\\"+ + "/\2\u01cb\u01cc\7\5\2\2\u01cc\u01d1\5\\/\2\u01cd\u01ce\7\5\2\2\u01ce\u01d0"+ + "\5\\/\2\u01cf\u01cd\3\2\2\2\u01d0\u01d3\3\2\2\2\u01d1\u01cf\3\2\2\2\u01d1"+ + "\u01d2\3\2\2\2\u01d2\u01d4\3\2\2\2\u01d3\u01d1\3\2\2\2\u01d4\u01d5\7\4"+ + "\2\2\u01d5\u01d8\3\2\2\2\u01d6\u01d8\5\60\31\2\u01d7\u01a6\3\2\2\2\u01d7"+ + "\u01a9\3\2\2\2\u01d7\u01ae\3\2\2\2\u01d7\u01ba\3\2\2\2\u01d7\u01c8\3\2"+ + "\2\2\u01d7\u01d6\3\2\2\2\u01d8\u01e1\3\2\2\2\u01d9\u01da\f\4\2\2\u01da"+ + "\u01db\7\n\2\2\u01db\u01e0\5.\30\5\u01dc\u01dd\f\3\2\2\u01dd\u01de\7\63"+ + "\2\2\u01de\u01e0\5.\30\4\u01df\u01d9\3\2\2\2\u01df\u01dc\3\2\2\2\u01e0"+ + "\u01e3\3\2\2\2\u01e1\u01df\3\2\2\2\u01e1\u01e2\3\2\2\2\u01e2/\3\2\2\2"+ + "\u01e3\u01e1\3\2\2\2\u01e4\u01e6\58\35\2\u01e5\u01e7\5\62\32\2\u01e6\u01e5"+ + "\3\2\2\2\u01e6\u01e7\3\2\2\2\u01e7\61\3\2\2\2\u01e8\u01ea\7/\2\2\u01e9"+ + "\u01e8\3\2\2\2\u01e9\u01ea\3\2\2\2\u01ea\u01eb\3\2\2\2\u01eb\u01ec\7\16"+ + "\2\2\u01ec\u01ed\58\35\2\u01ed\u01ee\7\n\2\2\u01ee\u01ef\58\35\2\u01ef"+ + "\u0217\3\2\2\2\u01f0\u01f2\7/\2\2\u01f1\u01f0\3\2\2\2\u01f1\u01f2\3\2"+ + "\2\2\u01f2\u01f3\3\2\2\2\u01f3\u01f4\7%\2\2\u01f4\u01f5\7\3\2\2\u01f5"+ + "\u01fa\5,\27\2\u01f6\u01f7\7\5\2\2\u01f7\u01f9\5,\27\2\u01f8\u01f6\3\2"+ + "\2\2\u01f9\u01fc\3\2\2\2\u01fa\u01f8\3\2\2\2\u01fa\u01fb\3\2\2\2\u01fb"+ + "\u01fd\3\2\2\2\u01fc\u01fa\3\2\2\2\u01fd\u01fe\7\4\2\2\u01fe\u0217\3\2"+ + "\2\2\u01ff\u0201\7/\2\2\u0200\u01ff\3\2\2\2\u0200\u0201\3\2\2\2\u0201"+ + "\u0202\3\2\2\2\u0202\u0203\7%\2\2\u0203\u0204\7\3\2\2\u0204\u0205\5\b"+ + "\5\2\u0205\u0206\7\4\2\2\u0206\u0217\3\2\2\2\u0207\u0209\7/\2\2\u0208"+ + "\u0207\3\2\2\2\u0208\u0209\3\2\2\2\u0209\u020a\3\2\2\2\u020a\u020b\7*"+ + "\2\2\u020b\u0217\5\64\33\2\u020c\u020e\7/\2\2\u020d\u020c\3\2\2\2\u020d"+ + "\u020e\3\2\2\2\u020e\u020f\3\2\2\2\u020f\u0210\7:\2\2\u0210\u0217\5\\"+ + "/\2\u0211\u0213\7\'\2\2\u0212\u0214\7/\2\2\u0213\u0212\3\2\2\2\u0213\u0214"+ + "\3\2\2\2\u0214\u0215\3\2\2\2\u0215\u0217\7\60\2\2\u0216\u01e9\3\2\2\2"+ + "\u0216\u01f1\3\2\2\2\u0216\u0200\3\2\2\2\u0216\u0208\3\2\2\2\u0216\u020d"+ + "\3\2\2\2\u0216\u0211\3\2\2\2\u0217\63\3\2\2\2\u0218\u021a\5\\/\2\u0219"+ + "\u021b\5\66\34\2\u021a\u0219\3\2\2\2\u021a\u021b\3\2\2\2\u021b\65\3\2"+ + "\2\2\u021c\u021d\7\30\2\2\u021d\u0223\5\\/\2\u021e\u021f\7J\2\2\u021f"+ + "\u0220\5\\/\2\u0220\u0221\7Q\2\2\u0221\u0223\3\2\2\2\u0222\u021c\3\2\2"+ + "\2\u0222\u021e\3\2\2\2\u0223\67\3\2\2\2\u0224\u0225\b\35\1\2\u0225\u0229"+ + "\5:\36\2\u0226\u0227\t\n\2\2\u0227\u0229\58\35\6\u0228\u0224\3\2\2\2\u0228"+ + "\u0226\3\2\2\2\u0229\u0236\3\2\2\2\u022a\u022b\f\5\2\2\u022b\u022c\t\13"+ + "\2\2\u022c\u0235\58\35\6\u022d\u022e\f\4\2\2\u022e\u022f\t\n\2\2\u022f"+ + "\u0235\58\35\5\u0230\u0231\f\3\2\2\u0231\u0232\5J&\2\u0232\u0233\58\35"+ + "\4\u0233\u0235\3\2\2\2\u0234\u022a\3\2\2\2\u0234\u022d\3\2\2\2\u0234\u0230"+ + "\3\2\2\2\u0235\u0238\3\2\2\2\u0236\u0234\3\2\2\2\u0236\u0237\3\2\2\2\u0237"+ + "9\3\2\2\2\u0238\u0236\3\2\2\2\u0239\u024f\5<\37\2\u023a\u024f\5@!\2\u023b"+ + "\u024f\5H%\2\u023c\u024f\7Z\2\2\u023d\u023e\5P)\2\u023e\u023f\7^\2\2\u023f"+ + "\u0241\3\2\2\2\u0240\u023d\3\2\2\2\u0240\u0241\3\2\2\2\u0241\u0242\3\2"+ + "\2\2\u0242\u024f\7Z\2\2\u0243\u024f\5D#\2\u0244\u0245\7\3\2\2\u0245\u0246"+ + "\5\b\5\2\u0246\u0247\7\4\2\2\u0247\u024f\3\2\2\2\u0248\u024f\5R*\2\u0249"+ + "\u024f\5P)\2\u024a\u024b\7\3\2\2\u024b\u024c\5,\27\2\u024c\u024d\7\4\2"+ + "\2\u024d\u024f\3\2\2\2\u024e\u0239\3\2\2\2\u024e\u023a\3\2\2\2\u024e\u023b"+ + "\3\2\2\2\u024e\u023c\3\2\2\2\u024e\u0240\3\2\2\2\u024e\u0243\3\2\2\2\u024e"+ + "\u0244\3\2\2\2\u024e\u0248\3\2\2\2\u024e\u0249\3\2\2\2\u024e\u024a\3\2"+ + "\2\2\u024f;\3\2\2\2\u0250\u0256\5> \2\u0251\u0252\7K\2\2\u0252\u0253\5"+ + "> \2\u0253\u0254\7Q\2\2\u0254\u0256\3\2\2\2\u0255\u0250\3\2\2\2\u0255"+ + "\u0251\3\2\2\2\u0256=\3\2\2\2\u0257\u0258\7\20\2\2\u0258\u0259\7\3\2\2"+ + "\u0259\u025a\5,\27\2\u025a\u025b\7\f\2\2\u025b\u025c\5N(\2\u025c\u025d"+ + "\7\4\2\2\u025d?\3\2\2\2\u025e\u0264\5B\"\2\u025f\u0260\7K\2\2\u0260\u0261"+ + "\5B\"\2\u0261\u0262\7Q\2\2\u0262\u0264\3\2\2\2\u0263\u025e\3\2\2\2\u0263"+ + "\u025f\3\2\2\2\u0264A\3\2\2\2\u0265\u0266\7\34\2\2\u0266\u0267\7\3\2\2"+ + "\u0267\u0268\5R*\2\u0268\u0269\7\37\2\2\u0269\u026a\58\35\2\u026a\u026b"+ + "\7\4\2\2\u026bC\3\2\2\2\u026c\u0272\5F$\2\u026d\u026e\7K\2\2\u026e\u026f"+ + "\5F$\2\u026f\u0270\7Q\2\2\u0270\u0272\3\2\2\2\u0271\u026c\3\2\2\2\u0271"+ + "\u026d\3\2\2\2\u0272E\3\2\2\2\u0273\u0274\5R*\2\u0274\u0280\7\3\2\2\u0275"+ + "\u0277\5\36\20\2\u0276\u0275\3\2\2\2\u0276\u0277\3\2\2\2\u0277\u0278\3"+ + "\2\2\2\u0278\u027d\5,\27\2\u0279\u027a\7\5\2\2\u027a\u027c\5,\27\2\u027b"+ + "\u0279\3\2\2\2\u027c\u027f\3\2\2\2\u027d\u027b\3\2\2\2\u027d\u027e\3\2"+ + "\2\2\u027e\u0281\3\2\2\2\u027f\u027d\3\2\2\2\u0280\u0276\3\2\2\2\u0280"+ + "\u0281\3\2\2\2\u0281\u0282\3\2\2\2\u0282\u0283\7\4\2\2\u0283G\3\2\2\2"+ + "\u0284\u029e\7\60\2\2\u0285\u029e\5Z.\2\u0286\u029e\5L\'\2\u0287\u0289"+ + "\7`\2\2\u0288\u0287\3\2\2\2\u0289\u028a\3\2\2\2\u028a\u0288\3\2\2\2\u028a"+ + "\u028b\3\2\2\2\u028b\u029e\3\2\2\2\u028c\u029e\7_\2\2\u028d\u028e\7M\2"+ + "\2\u028e\u028f\5\\/\2\u028f\u0290\7Q\2\2\u0290\u029e\3\2\2\2\u0291\u0292"+ + "\7N\2\2\u0292\u0293\5\\/\2\u0293\u0294\7Q\2\2\u0294\u029e\3\2\2\2\u0295"+ + "\u0296\7O\2\2\u0296\u0297\5\\/\2\u0297\u0298\7Q\2\2\u0298\u029e\3\2\2"+ + "\2\u0299\u029a\7P\2\2\u029a\u029b\5\\/\2\u029b\u029c\7Q\2\2\u029c\u029e"+ + "\3\2\2\2\u029d\u0284\3\2\2\2\u029d\u0285\3\2\2\2\u029d\u0286\3\2\2\2\u029d"+ + "\u0288\3\2\2\2\u029d\u028c\3\2\2\2\u029d\u028d\3\2\2\2\u029d\u0291\3\2"+ + "\2\2\u029d\u0295\3\2\2\2\u029d\u0299\3\2\2\2\u029eI\3\2\2\2\u029f\u02a0"+ + "\t\f\2\2\u02a0K\3\2\2\2\u02a1\u02a2\t\r\2\2\u02a2M\3\2\2\2\u02a3\u02a4"+ + "\5R*\2\u02a4O\3\2\2\2\u02a5\u02a6\5R*\2\u02a6\u02a7\7^\2\2\u02a7\u02a9"+ + "\3\2\2\2\u02a8\u02a5\3\2\2\2\u02a9\u02ac\3\2\2\2\u02aa\u02a8\3\2\2\2\u02aa"+ + "\u02ab\3\2\2\2\u02ab\u02ad\3\2\2\2\u02ac\u02aa\3\2\2\2\u02ad\u02ae\5R"+ + "*\2\u02aeQ\3\2\2\2\u02af\u02b2\5V,\2\u02b0\u02b2\5X-\2\u02b1\u02af\3\2"+ + "\2\2\u02b1\u02b0\3\2\2\2\u02b2S\3\2\2\2\u02b3\u02b4\5R*\2\u02b4\u02b5"+ + "\7\6\2\2\u02b5\u02b7\3\2\2\2\u02b6\u02b3\3\2\2\2\u02b6\u02b7\3\2\2\2\u02b7"+ + "\u02b8\3\2\2\2\u02b8\u02c0\7e\2\2\u02b9\u02ba\5R*\2\u02ba\u02bb\7\6\2"+ + "\2\u02bb\u02bd\3\2\2\2\u02bc\u02b9\3\2\2\2\u02bc\u02bd\3\2\2\2\u02bd\u02be"+ + "\3\2\2\2\u02be\u02c0\5R*\2\u02bf\u02b6\3\2\2\2\u02bf\u02bc\3\2\2\2\u02c0"+ + "U\3\2\2\2\u02c1\u02c4\7f\2\2\u02c2\u02c4\7g\2\2\u02c3\u02c1\3\2\2\2\u02c3"+ + "\u02c2\3\2\2\2\u02c4W\3\2\2\2\u02c5\u02c9\7c\2\2\u02c6\u02c9\5^\60\2\u02c7"+ + "\u02c9\7d\2\2\u02c8\u02c5\3\2\2\2\u02c8\u02c6\3\2\2\2\u02c8\u02c7\3\2"+ + "\2\2\u02c9Y\3\2\2\2\u02ca\u02cd\7b\2\2\u02cb\u02cd\7a\2\2\u02cc\u02ca"+ + "\3\2\2\2\u02cc\u02cb\3\2\2\2\u02cd[\3\2\2\2\u02ce\u02cf\t\16\2\2\u02cf"+ + "]\3\2\2\2\u02d0\u02d1\t\17\2\2\u02d1_\3\2\2\2eoqu~\u0080\u0084\u008a\u008d"+ + "\u0098\u009b\u00a5\u00a8\u00ab\u00ae\u00b6\u00b9\u00bf\u00c3\u00c6\u00c9"+ + "\u00cc\u00d3\u00db\u00de\u00ea\u00ed\u00f0\u00f7\u00fe\u0102\u0106\u010d"+ + "\u0111\u0115\u011a\u011e\u0126\u012a\u0131\u013c\u013f\u0143\u014f\u0152"+ + "\u0158\u015f\u0166\u0169\u016d\u0171\u0175\u0177\u0182\u0187\u018b\u018e"+ + "\u0194\u0197\u019d\u01a0\u01a2\u01b5\u01c3\u01d1\u01d7\u01df\u01e1\u01e6"+ + "\u01e9\u01f1\u01fa\u0200\u0208\u020d\u0213\u0216\u021a\u0222\u0228\u0234"+ + "\u0236\u0240\u024e\u0255\u0263\u0271\u0276\u027d\u0280\u028a\u029d\u02aa"+ + "\u02b1\u02b6\u02bc\u02bf\u02c3\u02c8\u02cc"; public static final ATN _ATN = new ATNDeserializer().deserialize(_serializedATN.toCharArray()); static { diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseVisitor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseVisitor.java index 35ce6cd0029d6..6745b3fa89ba9 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseVisitor.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlBaseVisitor.java @@ -1,8 +1,3 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ // ANTLR GENERATED CODE: DO NOT EDIT package org.elasticsearch.xpack.sql.parser; import org.antlr.v4.runtime.tree.ParseTreeVisitor; @@ -123,6 +118,12 @@ interface SqlBaseVisitor extends ParseTreeVisitor { * @return the visitor result */ T visitQueryNoWith(SqlBaseParser.QueryNoWithContext ctx); + /** + * Visit a parse tree produced by {@link SqlBaseParser#limitClause}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitLimitClause(SqlBaseParser.LimitClauseContext ctx); /** * Visit a parse tree produced by the {@code queryPrimaryDefault} * labeled alternative in {@link SqlBaseParser#queryTerm}. @@ -311,6 +312,12 @@ interface SqlBaseVisitor extends ParseTreeVisitor { * @return the visitor result */ T visitPattern(SqlBaseParser.PatternContext ctx); + /** + * Visit a parse tree produced by {@link SqlBaseParser#patternEscape}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitPatternEscape(SqlBaseParser.PatternEscapeContext ctx); /** * Visit a parse tree produced by the {@code valueExpressionDefault} * labeled alternative in {@link SqlBaseParser#valueExpression}. @@ -368,12 +375,12 @@ interface SqlBaseVisitor extends ParseTreeVisitor { */ T visitStar(SqlBaseParser.StarContext ctx); /** - * Visit a parse tree produced by the {@code functionCall} + * Visit a parse tree produced by the {@code function} * labeled alternative in {@link SqlBaseParser#primaryExpression}. * @param ctx the parse tree * @return the visitor result */ - T visitFunctionCall(SqlBaseParser.FunctionCallContext ctx); + T visitFunction(SqlBaseParser.FunctionContext ctx); /** * Visit a parse tree produced by the {@code subqueryExpression} * labeled alternative in {@link SqlBaseParser#primaryExpression}. @@ -402,6 +409,42 @@ interface SqlBaseVisitor extends ParseTreeVisitor { * @return the visitor result */ T visitParenthesizedExpression(SqlBaseParser.ParenthesizedExpressionContext ctx); + /** + * Visit a parse tree produced by {@link SqlBaseParser#castExpression}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitCastExpression(SqlBaseParser.CastExpressionContext ctx); + /** + * Visit a parse tree produced by {@link SqlBaseParser#castTemplate}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitCastTemplate(SqlBaseParser.CastTemplateContext ctx); + /** + * Visit a parse tree produced by {@link SqlBaseParser#extractExpression}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitExtractExpression(SqlBaseParser.ExtractExpressionContext ctx); + /** + * Visit a parse tree produced by {@link SqlBaseParser#extractTemplate}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitExtractTemplate(SqlBaseParser.ExtractTemplateContext ctx); + /** + * Visit a parse tree produced by {@link SqlBaseParser#functionExpression}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitFunctionExpression(SqlBaseParser.FunctionExpressionContext ctx); + /** + * Visit a parse tree produced by {@link SqlBaseParser#functionTemplate}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitFunctionTemplate(SqlBaseParser.FunctionTemplateContext ctx); /** * Visit a parse tree produced by the {@code nullLiteral} * labeled alternative in {@link SqlBaseParser#constant}. @@ -437,6 +480,34 @@ interface SqlBaseVisitor extends ParseTreeVisitor { * @return the visitor result */ T visitParamLiteral(SqlBaseParser.ParamLiteralContext ctx); + /** + * Visit a parse tree produced by the {@code dateEscapedLiteral} + * labeled alternative in {@link SqlBaseParser#constant}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitDateEscapedLiteral(SqlBaseParser.DateEscapedLiteralContext ctx); + /** + * Visit a parse tree produced by the {@code timeEscapedLiteral} + * labeled alternative in {@link SqlBaseParser#constant}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitTimeEscapedLiteral(SqlBaseParser.TimeEscapedLiteralContext ctx); + /** + * Visit a parse tree produced by the {@code timestampEscapedLiteral} + * labeled alternative in {@link SqlBaseParser#constant}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitTimestampEscapedLiteral(SqlBaseParser.TimestampEscapedLiteralContext ctx); + /** + * Visit a parse tree produced by the {@code guidEscapedLiteral} + * labeled alternative in {@link SqlBaseParser#constant}. + * @param ctx the parse tree + * @return the visitor result + */ + T visitGuidEscapedLiteral(SqlBaseParser.GuidEscapedLiteralContext ctx); /** * Visit a parse tree produced by {@link SqlBaseParser#comparisonOperator}. * @param ctx the parse tree diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlParser.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlParser.java index b7fe9178f911f..2824b5502a860 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlParser.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/parser/SqlParser.java @@ -33,10 +33,13 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.function.BiFunction; import java.util.function.Function; +import static java.lang.String.format; + public class SqlParser { private static final Logger log = Loggers.getLogger(SqlParser.class); @@ -102,16 +105,30 @@ private T invokeParser(String sql, List params, Function if (DEBUG) { debug(parser); + tokenStream.fill(); + + for (Token t : tokenStream.getTokens()) { + String symbolicName = SqlBaseLexer.VOCABULARY.getSymbolicName(t.getType()); + String literalName = SqlBaseLexer.VOCABULARY.getLiteralName(t.getType()); + log.info(format(Locale.ROOT, " %-15s '%s'", + symbolicName == null ? literalName : symbolicName, + t.getText())); + }; } ParserRuleContext tree = parseFunction.apply(parser); + if (DEBUG) { + log.info("Parse tree {} " + tree.toStringTree()); + } + return visitor.apply(new AstBuilder(paramTokens), tree); } private void debug(SqlBaseParser parser) { + // when debugging, use the exact prediction mode (needed for diagnostics as well) - parser.getInterpreter().setPredictionMode(PredictionMode.LL_EXACT_AMBIG_DETECTION); + parser.getInterpreter().setPredictionMode(PredictionMode.SLL); parser.addParseListener(parser.new TraceListener()); diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/EscapedFunctionsTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/EscapedFunctionsTests.java new file mode 100644 index 0000000000000..11ad24582efb6 --- /dev/null +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/parser/EscapedFunctionsTests.java @@ -0,0 +1,237 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.parser; + +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; +import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.Literal; +import org.elasticsearch.xpack.sql.expression.UnresolvedAttribute; +import org.elasticsearch.xpack.sql.expression.function.Function; +import org.elasticsearch.xpack.sql.expression.function.UnresolvedFunction; +import org.elasticsearch.xpack.sql.expression.regex.Like; +import org.elasticsearch.xpack.sql.expression.regex.LikePattern; +import org.elasticsearch.xpack.sql.plan.logical.Limit; +import org.elasticsearch.xpack.sql.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.sql.plan.logical.With; +import org.elasticsearch.xpack.sql.proto.SqlTypedParamValue; +import org.elasticsearch.xpack.sql.type.DataType; +import org.junit.Assert; + +import java.util.List; +import java.util.Locale; + +import static java.lang.String.format; +import static java.util.Arrays.asList; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.is; + +public class EscapedFunctionsTests extends ESTestCase { + + private final SqlParser parser = new SqlParser(); + + private Literal dateLiteral(String date) { + Expression exp = parser.createExpression(format(Locale.ROOT, "{d '%s'}", date)); + assertThat(exp, instanceOf(Expression.class)); + return (Literal) exp; + } + + private Literal timeLiteral(String date) { + Expression exp = parser.createExpression(format(Locale.ROOT, "{t '%s'}", date)); + assertThat(exp, instanceOf(Expression.class)); + return (Literal) exp; + } + + private Literal timestampLiteral(String date) { + Expression exp = parser.createExpression(format(Locale.ROOT, "{ts '%s'}", date)); + assertThat(exp, instanceOf(Expression.class)); + return (Literal) exp; + } + + private Literal guidLiteral(String date) { + Expression exp = parser.createExpression(format(Locale.ROOT, "{guid '%s'}", date)); + assertThat(exp, instanceOf(Expression.class)); + return (Literal) exp; + } + + private Limit limit(int limit) { + LogicalPlan plan = parser.createStatement(format(Locale.ROOT, "SELECT * FROM emp {limit %d}", limit)); + assertThat(plan, instanceOf(With.class)); + With with = (With) plan; + Limit limitPlan = (Limit) (with.child()); + assertThat(limitPlan.limit(), instanceOf(Literal.class)); + return limitPlan; + } + + private LikePattern likeEscape(String like, String character) { + Expression exp = parser.createExpression(format(Locale.ROOT, "exp LIKE '%s' {escape '%s'}", like, character)); + assertThat(exp, instanceOf(Like.class)); + return ((Like) exp).right(); + } + + private Function function(String name) { + Expression exp = parser.createExpression(format(Locale.ROOT, "{fn %s}", name)); + assertThat(exp, instanceOf(Function.class)); + return (Function) exp; + } + + public void testFunctionNoArg() { + Function f = function("SCORE()"); + assertEquals("SCORE", f.functionName()); + } + + public void testFunctionOneArg() { + Function f = function("ABS(foo)"); + assertEquals("ABS", f.functionName()); + assertEquals(1, f.arguments().size()); + Expression arg = f.arguments().get(0); + assertThat(arg, instanceOf(UnresolvedAttribute.class)); + UnresolvedAttribute ua = (UnresolvedAttribute) arg; + assertThat(ua.name(), is("foo")); + } + + public void testFunctionOneArgFunction() { + Function f = function("ABS({fn SCORE()})"); + assertEquals("ABS", f.functionName()); + assertEquals(1, f.arguments().size()); + Expression arg = f.arguments().get(0); + assertThat(arg, instanceOf(UnresolvedFunction.class)); + UnresolvedFunction uf = (UnresolvedFunction) arg; + assertThat(uf.name(), is("SCORE")); + } + + public void testFunctionFloorWithExtract() { + Function f = function("CAST({fn FLOOR({fn EXTRACT(YEAR FROM \"foo\")})} AS int)"); + assertEquals("CAST", f.functionName()); + assertEquals(1, f.arguments().size()); + Expression arg = f.arguments().get(0); + assertThat(arg, instanceOf(UnresolvedFunction.class)); + f = (Function) arg; + assertEquals("FLOOR", f.functionName()); + assertEquals(1, f.arguments().size()); + arg = f.arguments().get(0); + assertThat(arg, instanceOf(UnresolvedFunction.class)); + UnresolvedFunction uf = (UnresolvedFunction) arg; + assertThat(uf.name(), is("YEAR")); + } + + public void testFunctionWithFunctionWithArg() { + Function f = function("POWER(foo, {fn POWER({fn SCORE()}, {fN SCORE()})})"); + assertEquals("POWER", f.functionName()); + assertEquals(2, f.arguments().size()); + Expression arg = f.arguments().get(1); + assertThat(arg, instanceOf(UnresolvedFunction.class)); + UnresolvedFunction uf = (UnresolvedFunction) arg; + assertThat(uf.name(), is("POWER")); + assertEquals(2, uf.arguments().size()); + + List args = uf.arguments(); + arg = args.get(0); + assertThat(arg, instanceOf(UnresolvedFunction.class)); + uf = (UnresolvedFunction) arg; + assertThat(uf.name(), is("SCORE")); + + arg = args.get(1); + assertThat(arg, instanceOf(UnresolvedFunction.class)); + uf = (UnresolvedFunction) arg; + assertThat(uf.name(), is("SCORE")); + } + + public void testFunctionWithFunctionWithArgAndParams() { + Function f = (Function) parser.createExpression("POWER(?, {fn POWER({fn ABS(?)}, {fN ABS(?)})})", + asList(new SqlTypedParamValue(DataType.LONG, 1), + new SqlTypedParamValue(DataType.LONG, 1), + new SqlTypedParamValue(DataType.LONG, 1))); + + assertEquals("POWER", f.functionName()); + assertEquals(2, f.arguments().size()); + Expression arg = f.arguments().get(1); + assertThat(arg, instanceOf(UnresolvedFunction.class)); + UnresolvedFunction uf = (UnresolvedFunction) arg; + assertThat(uf.name(), is("POWER")); + assertEquals(2, uf.arguments().size()); + + List args = uf.arguments(); + arg = args.get(0); + assertThat(arg, instanceOf(UnresolvedFunction.class)); + uf = (UnresolvedFunction) arg; + assertThat(uf.name(), is("ABS")); + + arg = args.get(1); + assertThat(arg, instanceOf(UnresolvedFunction.class)); + uf = (UnresolvedFunction) arg; + assertThat(uf.name(), is("ABS")); + } + + public void testDateLiteral() { + Literal l = dateLiteral("2012-01-01"); + assertThat(l.dataType(), is(DataType.DATE)); + } + + public void testDateLiteralValidation() { + ParsingException ex = expectThrows(ParsingException.class, () -> dateLiteral("2012-13-01")); + assertEquals("line 1:2: Invalid date received; Cannot parse \"2012-13-01\": Value 13 for monthOfYear must be in the range [1,12]", + ex.getMessage()); + } + + public void testTimeLiteralUnsupported() { + SqlIllegalArgumentException ex = expectThrows(SqlIllegalArgumentException.class, () -> timeLiteral("10:10:10")); + assertThat(ex.getMessage(), is("Time (only) literals are not supported; a date component is required as well")); + } + + public void testTimeLiteralValidation() { + ParsingException ex = expectThrows(ParsingException.class, () -> timeLiteral("10:10:65")); + assertEquals("line 1:2: Invalid time received; Cannot parse \"10:10:65\": Value 65 for secondOfMinute must be in the range [0,59]", + ex.getMessage()); + } + + public void testTimestampLiteral() { + Literal l = timestampLiteral("2012-01-01 10:01:02.3456"); + assertThat(l.dataType(), is(DataType.DATE)); + } + + public void testTimestampLiteralValidation() { + ParsingException ex = expectThrows(ParsingException.class, () -> timestampLiteral("2012-01-01T10:01:02.3456")); + assertEquals( + "line 1:2: Invalid timestamp received; Invalid format: \"2012-01-01T10:01:02.3456\" is malformed at \"T10:01:02.3456\"", + ex.getMessage()); + } + + public void testGUID() { + Literal l = guidLiteral("12345678-90ab-cdef-0123-456789abcdef"); + assertThat(l.dataType(), is(DataType.KEYWORD)); + + l = guidLiteral("12345678-90AB-cdef-0123-456789ABCdef"); + assertThat(l.dataType(), is(DataType.KEYWORD)); + } + + public void testGUIDValidationHexa() { + ParsingException ex = expectThrows(ParsingException.class, () -> guidLiteral("12345678-90ab-cdef-0123-456789abcdeH")); + assertEquals("line 1:8: Invalid GUID, expected hexadecimal at offset[35], found [H]", ex.getMessage()); + } + + public void testGUIDValidationGroups() { + ParsingException ex = expectThrows(ParsingException.class, () -> guidLiteral("12345678A90ab-cdef-0123-456789abcdeH")); + assertEquals("line 1:8: Invalid GUID, expected group separator at offset [8], found [A]", ex.getMessage()); + } + + public void testGUIDValidationLength() { + ParsingException ex = expectThrows(ParsingException.class, () -> guidLiteral("12345678A90")); + assertEquals("line 1:8: Invalid GUID, too short", ex.getMessage()); + } + + + public void testLimit() { + Limit limit = limit(10); + Literal l = (Literal) limit.limit(); + Assert.assertThat(l.value(), is(10)); + } + + public void testLikeEscape() { + LikePattern pattern = likeEscape("|%tring", "|"); + assertThat(pattern.escape(), is('|')); + } +} \ No newline at end of file From a0716727ecdebfc17fcb9405452608ec31223a14 Mon Sep 17 00:00:00 2001 From: Armin Braun Date: Thu, 12 Jul 2018 07:45:11 +0200 Subject: [PATCH 36/50] Fix assertIngestDocument wrongfully passing (#31913) (#31951) * Fix assertIngestDocument wrongfully passing * Previously docA being subset of docB passed because iteration was over docA's keys only * Scalars in nested fields were not compared in all cases * Assertion errors were hard to interpret (message wasn't correct since it only mentioned the class type) * In cases where two paths contained different types a ClassCastException was thrown instead of an AssertionError * Fixes #28492 --- .../ingest/common/GrokProcessorTests.java | 2 +- .../ingest/common/JsonProcessorTests.java | 14 ++-- .../ingest/IngestDocumentMatcher.java | 57 ++++++++----- .../ingest/IngestDocumentMatcherTests.java | 82 +++++++++++++++++++ 4 files changed, 123 insertions(+), 32 deletions(-) create mode 100644 test/framework/src/test/java/org/elasticsearch/ingest/IngestDocumentMatcherTests.java diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorTests.java index 0eba79523aca2..68654923ae926 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/GrokProcessorTests.java @@ -129,7 +129,7 @@ public void testMissingField() { public void testMissingFieldWithIgnoreMissing() throws Exception { String fieldName = "foo.bar"; IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); - IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>()); + IngestDocument ingestDocument = new IngestDocument(originalIngestDocument); GrokProcessor processor = new GrokProcessor(randomAlphaOfLength(10), Collections.singletonMap("ONE", "1"), Collections.singletonList("%{ONE:one}"), fieldName, false, true, ThreadWatchdog.noop()); processor.execute(ingestDocument); diff --git a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java index 245285259b47a..2867ed1d24053 100644 --- a/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java +++ b/modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/JsonProcessorTests.java @@ -33,7 +33,6 @@ import java.util.List; import java.util.Map; -import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; @@ -55,7 +54,7 @@ public void testExecute() throws Exception { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); jsonProcessor.execute(ingestDocument); Map jsonified = ingestDocument.getFieldValue(randomTargetField, Map.class); - assertIngestDocument(ingestDocument.getFieldValue(randomTargetField, Object.class), jsonified); + assertEquals(ingestDocument.getFieldValue(randomTargetField, Object.class), jsonified); } public void testInvalidValue() { @@ -161,13 +160,10 @@ public void testAddToRoot() throws Exception { IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document); jsonProcessor.execute(ingestDocument); - Map expected = new HashMap<>(); - expected.put("a", 1); - expected.put("b", 2); - expected.put("c", "see"); - IngestDocument expectedIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), expected); - - assertIngestDocument(ingestDocument, expectedIngestDocument); + Map sourceAndMetadata = ingestDocument.getSourceAndMetadata(); + assertEquals(1, sourceAndMetadata.get("a")); + assertEquals(2, sourceAndMetadata.get("b")); + assertEquals("see", sourceAndMetadata.get("c")); } public void testAddBoolToRoot() { diff --git a/test/framework/src/main/java/org/elasticsearch/ingest/IngestDocumentMatcher.java b/test/framework/src/main/java/org/elasticsearch/ingest/IngestDocumentMatcher.java index bb058c5cfdbb5..a7a8a3f11b1be 100644 --- a/test/framework/src/main/java/org/elasticsearch/ingest/IngestDocumentMatcher.java +++ b/test/framework/src/main/java/org/elasticsearch/ingest/IngestDocumentMatcher.java @@ -20,48 +20,61 @@ package org.elasticsearch.ingest; import java.util.List; -import java.util.Locale; import java.util.Map; - -import static org.hamcrest.Matchers.equalTo; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertThat; +import java.util.Objects; public class IngestDocumentMatcher { /** * Helper method to assert the equivalence between two IngestDocuments. * - * @param a first object to compare - * @param b second object to compare + * @param docA first document to compare + * @param docB second document to compare */ - public static void assertIngestDocument(Object a, Object b) { + public static void assertIngestDocument(IngestDocument docA, IngestDocument docB) { + if ((deepEquals(docA.getIngestMetadata(), docB.getIngestMetadata(), true) && + deepEquals(docA.getSourceAndMetadata(), docB.getSourceAndMetadata(), false)) == false) { + throw new AssertionError("Expected [" + docA + "] but received [" + docB + "]."); + } + } + + private static boolean deepEquals(Object a, Object b, boolean isIngestMeta) { if (a instanceof Map) { Map mapA = (Map) a; + if (b instanceof Map == false) { + return false; + } Map mapB = (Map) b; + if (mapA.size() != mapB.size()) { + return false; + } for (Map.Entry entry : mapA.entrySet()) { - if (entry.getValue() instanceof List || entry.getValue() instanceof Map) { - assertIngestDocument(entry.getValue(), mapB.get(entry.getKey())); + Object key = entry.getKey(); + // Don't compare the timestamp of ingest metadata since it will differ between executions + if ((isIngestMeta && "timestamp".equals(key)) == false + && deepEquals(entry.getValue(), mapB.get(key), false) == false) { + return false; } } + return true; } else if (a instanceof List) { List listA = (List) a; + if (b instanceof List == false) { + return false; + } List listB = (List) b; - for (int i = 0; i < listA.size(); i++) { + int countA = listA.size(); + if (countA != listB.size()) { + return false; + } + for (int i = 0; i < countA; i++) { Object value = listA.get(i); - if (value instanceof List || value instanceof Map) { - assertIngestDocument(value, listB.get(i)); + if (deepEquals(value, listB.get(i), false) == false) { + return false; } } - } else if (a instanceof byte[]) { - assertArrayEquals((byte[]) a, (byte[])b); - } else if (a instanceof IngestDocument) { - IngestDocument docA = (IngestDocument) a; - IngestDocument docB = (IngestDocument) b; - assertIngestDocument(docA.getSourceAndMetadata(), docB.getSourceAndMetadata()); - assertIngestDocument(docA.getIngestMetadata(), docB.getIngestMetadata()); + return true; } else { - String msg = String.format(Locale.ROOT, "Expected %s class to be equal to %s", a.getClass().getName(), b.getClass().getName()); - assertThat(msg, a, equalTo(b)); + return Objects.deepEquals(a, b); } } } diff --git a/test/framework/src/test/java/org/elasticsearch/ingest/IngestDocumentMatcherTests.java b/test/framework/src/test/java/org/elasticsearch/ingest/IngestDocumentMatcherTests.java new file mode 100644 index 0000000000000..bff9a923b9f75 --- /dev/null +++ b/test/framework/src/test/java/org/elasticsearch/ingest/IngestDocumentMatcherTests.java @@ -0,0 +1,82 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.ingest; + +import org.elasticsearch.test.ESTestCase; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument; + +public class IngestDocumentMatcherTests extends ESTestCase { + + public void testDifferentMapData() { + Map sourceAndMetadata1 = new HashMap<>(); + sourceAndMetadata1.put("foo", "bar"); + IngestDocument document1 = new IngestDocument(sourceAndMetadata1, new HashMap<>()); + IngestDocument document2 = new IngestDocument(new HashMap<>(), new HashMap<>()); + assertThrowsOnComparision(document1, document2); + } + + public void testDifferentLengthListData() { + String rootKey = "foo"; + IngestDocument document1 = + new IngestDocument(Collections.singletonMap(rootKey, Arrays.asList("bar", "baz")), new HashMap<>()); + IngestDocument document2 = + new IngestDocument(Collections.singletonMap(rootKey, Collections.emptyList()), new HashMap<>()); + assertThrowsOnComparision(document1, document2); + } + + public void testDifferentNestedListFieldData() { + String rootKey = "foo"; + IngestDocument document1 = + new IngestDocument(Collections.singletonMap(rootKey, Arrays.asList("bar", "baz")), new HashMap<>()); + IngestDocument document2 = + new IngestDocument(Collections.singletonMap(rootKey, Arrays.asList("bar", "blub")), new HashMap<>()); + assertThrowsOnComparision(document1, document2); + } + + public void testDifferentNestedMapFieldData() { + String rootKey = "foo"; + IngestDocument document1 = + new IngestDocument(Collections.singletonMap(rootKey, Collections.singletonMap("bar", "baz")), new HashMap<>()); + IngestDocument document2 = + new IngestDocument(Collections.singletonMap(rootKey, Collections.singletonMap("bar", "blub")), new HashMap<>()); + assertThrowsOnComparision(document1, document2); + } + + public void testOnTypeConflict() { + String rootKey = "foo"; + IngestDocument document1 = + new IngestDocument(Collections.singletonMap(rootKey, Collections.singletonList("baz")), new HashMap<>()); + IngestDocument document2 = new IngestDocument( + Collections.singletonMap(rootKey, Collections.singletonMap("blub", "blab")), new HashMap<>() + ); + assertThrowsOnComparision(document1, document2); + } + + private static void assertThrowsOnComparision(IngestDocument document1, IngestDocument document2) { + expectThrows(AssertionError.class, () -> assertIngestDocument(document1, document2)); + expectThrows(AssertionError.class, () -> assertIngestDocument(document2, document1)); + } +} From 94c1ebebe4b0f938d327026b182e0294cc7bfc86 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Thu, 12 Jul 2018 09:21:10 +0200 Subject: [PATCH 37/50] [Test] Reactive 3rd party tests on CI (#31919) 3rd party tests are failing because the repository-s3 is expecting some enviromnent variables in order to test session tokens but the CI job is not ready yet to provide those. This pull request relaxes the constraints on the presence of env vars so that the 3rd party tests can still be executed on CI. closes #31813 --- plugins/repository-s3/build.gradle | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle index dc2140a6086a4..5af0a412b4cc0 100644 --- a/plugins/repository-s3/build.gradle +++ b/plugins/repository-s3/build.gradle @@ -92,23 +92,26 @@ String s3TemporaryBasePath = System.getenv("amazon_s3_base_path_temporary") // If all these variables are missing then we are testing against the internal fixture instead, which has the following // credentials hard-coded in. -if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3PermanentBasePath - && !s3TemporaryAccessKey && !s3TemporarySecretKey && !s3TemporaryBucket && !s3TemporaryBasePath && !s3TemporarySessionToken) { - +if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3PermanentBasePath) { s3PermanentAccessKey = 's3_integration_test_permanent_access_key' s3PermanentSecretKey = 's3_integration_test_permanent_secret_key' s3PermanentBucket = 'permanent-bucket-test' s3PermanentBasePath = 'integration_test' + useFixture = true + +} else if (!s3PermanentAccessKey || !s3PermanentSecretKey || !s3PermanentBucket || !s3PermanentBasePath) { + throw new IllegalArgumentException("not all options specified to run against external S3 service") +} + +if (!s3TemporaryAccessKey && !s3TemporarySecretKey && !s3TemporaryBucket && !s3TemporaryBasePath && !s3TemporarySessionToken) { s3TemporaryAccessKey = 's3_integration_test_temporary_access_key' s3TemporarySecretKey = 's3_integration_test_temporary_secret_key' s3TemporaryBucket = 'temporary-bucket-test' s3TemporaryBasePath = 'integration_test' s3TemporarySessionToken = 's3_integration_test_temporary_session_token' - useFixture = true -} else if (!s3PermanentAccessKey || !s3PermanentSecretKey || !s3PermanentBucket || !s3PermanentBasePath - || !s3TemporaryAccessKey || !s3TemporarySecretKey || !s3TemporaryBucket || !s3TemporaryBasePath || !s3TemporarySessionToken) { +} else if (!s3TemporaryAccessKey || !s3TemporarySecretKey || !s3TemporaryBucket || !s3TemporaryBasePath || !s3TemporarySessionToken) { throw new IllegalArgumentException("not all options specified to run against external S3 service") } @@ -296,6 +299,13 @@ processTestResources { MavenFilteringHack.filter(it, expansions) } +project.afterEvaluate { + if (useFixture == false) { + // 30_repository_temporary_credentials is not ready for CI yet + integTestRunner.systemProperty 'tests.rest.blacklist', 'repository_s3/30_repository_temporary_credentials/*' + } +} + integTestCluster { keystoreSetting 's3.client.integration_test_permanent.access_key', s3PermanentAccessKey keystoreSetting 's3.client.integration_test_permanent.secret_key', s3PermanentSecretKey From a1304c660c98ef75c6020c2f1e51804d3f56cc6f Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Thu, 12 Jul 2018 08:41:07 +0000 Subject: [PATCH 38/50] Move build integration tests out of :buildSrc project (#31961) This way building buildSrc will not be delayed for all invocations, and will be less disruptive if integration tests fail --- buildSrc/build.gradle | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle index 4e31de08829cc..3d100daf7d65f 100644 --- a/buildSrc/build.gradle +++ b/buildSrc/build.gradle @@ -128,6 +128,10 @@ if (project == rootProject) { } mavenCentral() } + test { + include "**/*Tests.class" + exclude "**/*IT.class" + } } /***************************************************************************** @@ -152,6 +156,18 @@ if (project != rootProject) { jarHell.enabled = false thirdPartyAudit.enabled = false + // tests can't be run with randomized test runner + // it's fine as we run them as part of :buildSrc + test.enabled = false + task integTest(type: Test) { + exclude "**/*Tests.class" + include "**/*IT.class" + testClassesDirs = sourceSets.test.output.classesDirs + classpath = sourceSets.test.runtimeClasspath + inputs.dir(file("src/testKit")) + } + check.dependsOn(integTest) + // TODO: re-enable once randomizedtesting gradle code is published and removed from here licenseHeaders.enabled = false From b424cd1209b178cff88b249633f94e7f4c051cd7 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Thu, 12 Jul 2018 08:55:31 +0000 Subject: [PATCH 39/50] Work around reported problem in eclipse (#31960) The Gradle plugin https://docs.gradle.org/current/userguide/java_gradle_plugin.html added recently adds a folder to the CP, which is not created for the eclipse import, ausing eclipse to complain. --- build.gradle | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build.gradle b/build.gradle index e07270e0022dd..6ed121786cf2c 100644 --- a/build.gradle +++ b/build.gradle @@ -429,6 +429,9 @@ allprojects { if (isEclipse) { // set this so generated dirs will be relative to eclipse build project.buildDir = eclipseBuild + // Work around https://docs.gradle.org/current/userguide/java_gradle_plugin.html confusing Eclipse by the metadata + // it adds to the classpath + project.file("$buildDir/pluginUnderTestMetadata").mkdirs() } eclipse.classpath.file.whenMerged { classpath -> // give each source folder a unique corresponding output folder From e7602c671bf7751b85a631ac0c34b414b4d3d088 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Thu, 12 Jul 2018 11:51:33 +0200 Subject: [PATCH 40/50] muted tests due to #31940 --- .../org/elasticsearch/client/RestClientBuilderIntegTests.java | 3 +++ .../smoketest/SmokeTestMonitoringWithSecurityIT.java | 1 + 2 files changed, 4 insertions(+) diff --git a/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderIntegTests.java b/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderIntegTests.java index 93f8481bea662..30359ea90f666 100644 --- a/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderIntegTests.java +++ b/client/rest/src/test/java/org/elasticsearch/client/RestClientBuilderIntegTests.java @@ -72,6 +72,9 @@ public static void stopHttpServers() throws IOException { } public void testBuilderUsesDefaultSSLContext() throws Exception { + assumeFalse("Due to bug inside jdk, this test can't momentarily run with java 11. " + + "See: https://github.com/elastic/elasticsearch/issues/31940", + System.getProperty("java.version").contains("11")); final SSLContext defaultSSLContext = SSLContext.getDefault(); try { try (RestClient client = buildRestClient()) { diff --git a/x-pack/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestMonitoringWithSecurityIT.java b/x-pack/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestMonitoringWithSecurityIT.java index c427d8bf32c86..52c2a7dfa2d2d 100644 --- a/x-pack/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestMonitoringWithSecurityIT.java +++ b/x-pack/qa/smoke-test-plugins-ssl/src/test/java/org/elasticsearch/smoketest/SmokeTestMonitoringWithSecurityIT.java @@ -127,6 +127,7 @@ private boolean getMonitoringUsageExportersDefined() throws Exception { return monitoringUsage.get().getExporters().isEmpty() == false; } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/31940") public void testHTTPExporterWithSSL() throws Exception { // Ensures that the exporter is actually on assertBusy(() -> assertThat("[_http] exporter is not defined", getMonitoringUsageExportersDefined(), is(true))); From f50e58cb25a0e27cb71038f97086bac2521f5ff5 Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Thu, 12 Jul 2018 15:05:42 +0300 Subject: [PATCH 41/50] SQL: Add support for single parameter text manipulating functions (#31874) Added support for ASCII, BIT_LENGTH, CHAR, CHAR_LENGTH, LCASE, LENGTH, LTRIM, RTRIM, SPACE, UCASE functions. Wherever Painless scripting is necessary (WHERE conditions, ORDER BY etc), those scripts are being used. --- .../expression/function/FunctionRegistry.java | 22 +++ .../function/scalar/Processors.java | 3 + .../function/scalar/math/ATan2.java | 2 +- .../function/scalar/math/Power.java | 2 +- .../function/scalar/string/Ascii.java | 42 +++++ .../function/scalar/string/BitLength.java | 43 +++++ .../function/scalar/string/Char.java | 42 +++++ .../function/scalar/string/CharLength.java | 42 +++++ .../function/scalar/string/LCase.java | 42 +++++ .../function/scalar/string/LTrim.java | 43 +++++ .../function/scalar/string/Length.java | 43 +++++ .../function/scalar/string/RTrim.java | 43 +++++ .../function/scalar/string/Space.java | 42 +++++ .../scalar/string/StringFunctionUtils.java | 51 ++++++ .../scalar/string/StringProcessor.java | 160 ++++++++++++++++++ .../function/scalar/string/UCase.java | 43 +++++ .../scalar/string/UnaryStringFunction.java | 89 ++++++++++ .../scalar/string/UnaryStringIntFunction.java | 90 ++++++++++ .../whitelist/InternalSqlScriptUtils.java | 41 +++++ .../xpack/sql/planner/QueryTranslator.java | 3 + .../xpack/sql/util/StringUtils.java | 27 +++ .../xpack/sql/plugin/sql_whitelist.txt | 10 ++ .../string/StringFunctionProcessorTests.java | 160 ++++++++++++++++++ .../sql/planner/QueryTranslatorTests.java | 10 ++ .../xpack/qa/sql/cli/ShowTestCase.java | 3 + .../xpack/qa/sql/jdbc/CsvSpecTestCase.java | 1 + .../xpack/qa/sql/jdbc/SqlSpecTestCase.java | 1 + .../sql/src/main/resources/command.csv-spec | 11 ++ .../qa/sql/src/main/resources/docs.csv-spec | 13 +- .../sql/src/main/resources/functions.csv-spec | 30 ++++ .../main/resources/string-functions.sql-spec | 76 +++++++++ 31 files changed, 1227 insertions(+), 3 deletions(-) create mode 100644 x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Ascii.java create mode 100644 x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BitLength.java create mode 100644 x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Char.java create mode 100644 x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/CharLength.java create mode 100644 x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/LCase.java create mode 100644 x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/LTrim.java create mode 100644 x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Length.java create mode 100644 x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/RTrim.java create mode 100644 x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Space.java create mode 100644 x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionUtils.java create mode 100644 x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringProcessor.java create mode 100644 x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/UCase.java create mode 100644 x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/UnaryStringFunction.java create mode 100644 x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/UnaryStringIntFunction.java create mode 100644 x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionProcessorTests.java create mode 100644 x-pack/qa/sql/src/main/resources/functions.csv-spec create mode 100644 x-pack/qa/sql/src/main/resources/string-functions.sql-spec diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionRegistry.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionRegistry.java index 22141497a5c25..212149683ffd9 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionRegistry.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/FunctionRegistry.java @@ -58,6 +58,16 @@ import org.elasticsearch.xpack.sql.expression.function.scalar.math.Sinh; import org.elasticsearch.xpack.sql.expression.function.scalar.math.Sqrt; import org.elasticsearch.xpack.sql.expression.function.scalar.math.Tan; +import org.elasticsearch.xpack.sql.expression.function.scalar.string.Ascii; +import org.elasticsearch.xpack.sql.expression.function.scalar.string.BitLength; +import org.elasticsearch.xpack.sql.expression.function.scalar.string.Char; +import org.elasticsearch.xpack.sql.expression.function.scalar.string.CharLength; +import org.elasticsearch.xpack.sql.expression.function.scalar.string.LCase; +import org.elasticsearch.xpack.sql.expression.function.scalar.string.LTrim; +import org.elasticsearch.xpack.sql.expression.function.scalar.string.Length; +import org.elasticsearch.xpack.sql.expression.function.scalar.string.RTrim; +import org.elasticsearch.xpack.sql.expression.function.scalar.string.Space; +import org.elasticsearch.xpack.sql.expression.function.scalar.string.UCase; import org.elasticsearch.xpack.sql.parser.ParsingException; import org.elasticsearch.xpack.sql.tree.Location; import org.elasticsearch.xpack.sql.util.StringUtils; @@ -134,6 +144,17 @@ public class FunctionRegistry { def(Sinh.class, Sinh::new), def(Sqrt.class, Sqrt::new), def(Tan.class, Tan::new), + // String + def(Ascii.class, Ascii::new), + def(Char.class, Char::new), + def(BitLength.class, BitLength::new), + def(CharLength.class, CharLength::new), + def(LCase.class, LCase::new), + def(Length.class, Length::new), + def(LTrim.class, LTrim::new), + def(RTrim.class, RTrim::new), + def(Space.class, Space::new), + def(UCase.class, UCase::new), // Special def(Score.class, Score::new))); @@ -299,6 +320,7 @@ interface BinaryFunctionBuilder { T build(Location location, Expression lhs, Expression rhs); } + @SuppressWarnings("overloads") private static FunctionDefinition def(Class function, FunctionBuilder builder, boolean datetime, String... aliases) { String primaryName = normalize(function.getSimpleName()); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/Processors.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/Processors.java index 2084ad684df67..3c3f629cc1caf 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/Processors.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/Processors.java @@ -17,6 +17,7 @@ import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.ConstantProcessor; import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.HitExtractorProcessor; import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.Processor; +import org.elasticsearch.xpack.sql.expression.function.scalar.string.StringProcessor; import java.util.ArrayList; import java.util.List; @@ -46,6 +47,8 @@ public static List getNamedWriteables() { entries.add(new Entry(Processor.class, DateTimeProcessor.NAME, DateTimeProcessor::new)); // math entries.add(new Entry(Processor.class, MathProcessor.NAME, MathProcessor::new)); + // string + entries.add(new Entry(Processor.class, StringProcessor.NAME, StringProcessor::new)); return entries; } } \ No newline at end of file diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/ATan2.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/ATan2.java index 1a86a44d32b13..24bbebd64c2db 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/ATan2.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/ATan2.java @@ -30,7 +30,7 @@ protected BiFunction operation() { } @Override - protected NodeInfo info() { + protected NodeInfo info() { return NodeInfo.create(this, ATan2::new, left(), right()); } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Power.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Power.java index d38d7067cafb5..4e362dbb8e5eb 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Power.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/math/Power.java @@ -26,7 +26,7 @@ protected BiFunction operation() { } @Override - protected NodeInfo info() { + protected NodeInfo info() { return NodeInfo.create(this, Power::new, left(), right()); } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Ascii.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Ascii.java new file mode 100644 index 0000000000000..7f74a22cd80af --- /dev/null +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Ascii.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.string; + +import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.string.StringProcessor.StringOperation; +import org.elasticsearch.xpack.sql.tree.Location; +import org.elasticsearch.xpack.sql.tree.NodeInfo; +import org.elasticsearch.xpack.sql.type.DataType; + +/** + * Returns the ASCII code of the leftmost character of the given (char) expression. + */ +public class Ascii extends UnaryStringFunction { + + public Ascii(Location location, Expression field) { + super(location, field); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, Ascii::new, field()); + } + + @Override + protected Ascii replaceChild(Expression newChild) { + return new Ascii(location(), newChild); + } + + @Override + protected StringOperation operation() { + return StringOperation.ASCII; + } + + @Override + public DataType dataType() { + return DataType.INTEGER; + } +} diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BitLength.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BitLength.java new file mode 100644 index 0000000000000..3254e0538f06e --- /dev/null +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/BitLength.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.string; + +import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.string.StringProcessor.StringOperation; +import org.elasticsearch.xpack.sql.tree.Location; +import org.elasticsearch.xpack.sql.tree.NodeInfo; +import org.elasticsearch.xpack.sql.type.DataType; + +/** + * Returns returns the number of bits contained within the value expression. + */ +public class BitLength extends UnaryStringFunction { + + public BitLength(Location location, Expression field) { + super(location, field); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, BitLength::new, field()); + } + + @Override + protected BitLength replaceChild(Expression newChild) { + return new BitLength(location(), newChild); + } + + @Override + protected StringOperation operation() { + return StringOperation.BIT_LENGTH; + } + + @Override + public DataType dataType() { + //TODO investigate if a data type Long (BIGINT) wouldn't be more appropriate here + return DataType.INTEGER; + } +} \ No newline at end of file diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Char.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Char.java new file mode 100644 index 0000000000000..06d1c3d81cc42 --- /dev/null +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Char.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.string; + +import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.string.StringProcessor.StringOperation; +import org.elasticsearch.xpack.sql.tree.Location; +import org.elasticsearch.xpack.sql.tree.NodeInfo; +import org.elasticsearch.xpack.sql.type.DataType; + +/** + * Converts an int ASCII code to a character value. + */ +public class Char extends UnaryStringIntFunction { + + public Char(Location location, Expression field) { + super(location, field); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, Char::new, field()); + } + + @Override + protected Char replaceChild(Expression newChild) { + return new Char(location(), newChild); + } + + @Override + protected StringOperation operation() { + return StringOperation.CHAR; + } + + @Override + public DataType dataType() { + return DataType.KEYWORD; + } +} \ No newline at end of file diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/CharLength.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/CharLength.java new file mode 100644 index 0000000000000..bdf43fbeb4ee9 --- /dev/null +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/CharLength.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.string; + +import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.string.StringProcessor.StringOperation; +import org.elasticsearch.xpack.sql.tree.Location; +import org.elasticsearch.xpack.sql.tree.NodeInfo; +import org.elasticsearch.xpack.sql.type.DataType; + +/** + * Returns the length (in characters) of the string expression. + */ +public class CharLength extends UnaryStringFunction { + + public CharLength(Location location, Expression field) { + super(location, field); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, CharLength::new, field()); + } + + @Override + protected CharLength replaceChild(Expression newChild) { + return new CharLength(location(), newChild); + } + + @Override + protected StringOperation operation() { + return StringOperation.CHAR_LENGTH; + } + + @Override + public DataType dataType() { + return DataType.INTEGER; + } +} \ No newline at end of file diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/LCase.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/LCase.java new file mode 100644 index 0000000000000..a074fcb3b98b6 --- /dev/null +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/LCase.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.string; + +import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.string.StringProcessor.StringOperation; +import org.elasticsearch.xpack.sql.tree.Location; +import org.elasticsearch.xpack.sql.tree.NodeInfo; +import org.elasticsearch.xpack.sql.type.DataType; + +/** + * Lowercases all uppercase letters in a string. + */ +public class LCase extends UnaryStringFunction { + + public LCase(Location location, Expression field) { + super(location, field); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, LCase::new, field()); + } + + @Override + protected LCase replaceChild(Expression newChild) { + return new LCase(location(), newChild); + } + + @Override + protected StringOperation operation() { + return StringOperation.LCASE; + } + + @Override + public DataType dataType() { + return DataType.KEYWORD; + } +} diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/LTrim.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/LTrim.java new file mode 100644 index 0000000000000..616f8ccdfedfc --- /dev/null +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/LTrim.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.string; + +import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.string.StringProcessor.StringOperation; +import org.elasticsearch.xpack.sql.tree.Location; +import org.elasticsearch.xpack.sql.tree.NodeInfo; +import org.elasticsearch.xpack.sql.type.DataType; + +/** + * Trims the leading whitespaces. + */ +public class LTrim extends UnaryStringFunction { + + public LTrim(Location location, Expression field) { + super(location, field); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, LTrim::new, field()); + } + + @Override + protected LTrim replaceChild(Expression newChild) { + return new LTrim(location(), newChild); + } + + @Override + protected StringOperation operation() { + return StringOperation.LTRIM; + } + + @Override + public DataType dataType() { + return DataType.KEYWORD; + } + +} diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Length.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Length.java new file mode 100644 index 0000000000000..8e3efbfceec22 --- /dev/null +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Length.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.string; + +import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.string.StringProcessor.StringOperation; +import org.elasticsearch.xpack.sql.tree.Location; +import org.elasticsearch.xpack.sql.tree.NodeInfo; +import org.elasticsearch.xpack.sql.type.DataType; + +/** + * Returns the length (number of characters) in a string, excluding the trailing blanks. + */ +public class Length extends UnaryStringFunction { + + public Length(Location location, Expression field) { + super(location, field); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, Length::new, field()); + } + + @Override + protected Length replaceChild(Expression newChild) { + return new Length(location(), newChild); + } + + @Override + protected StringOperation operation() { + return StringOperation.LENGTH; + } + + @Override + public DataType dataType() { + return DataType.INTEGER; + } + +} diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/RTrim.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/RTrim.java new file mode 100644 index 0000000000000..433668420d35f --- /dev/null +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/RTrim.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.string; + +import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.string.StringProcessor.StringOperation; +import org.elasticsearch.xpack.sql.tree.Location; +import org.elasticsearch.xpack.sql.tree.NodeInfo; +import org.elasticsearch.xpack.sql.type.DataType; + +/** + * Trims the trailing whitespaces. + */ +public class RTrim extends UnaryStringFunction { + + public RTrim(Location location, Expression field) { + super(location, field); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, RTrim::new, field()); + } + + @Override + protected RTrim replaceChild(Expression newChild) { + return new RTrim(location(), newChild); + } + + @Override + protected StringOperation operation() { + return StringOperation.RTRIM; + } + + @Override + public DataType dataType() { + return DataType.KEYWORD; + } + +} diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Space.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Space.java new file mode 100644 index 0000000000000..37809482c219b --- /dev/null +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/Space.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.string; + +import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.string.StringProcessor.StringOperation; +import org.elasticsearch.xpack.sql.tree.Location; +import org.elasticsearch.xpack.sql.tree.NodeInfo; +import org.elasticsearch.xpack.sql.type.DataType; + +/** + * Generates a string consisting of count spaces. + */ +public class Space extends UnaryStringIntFunction { + + public Space(Location location, Expression field) { + super(location, field); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, Space::new, field()); + } + + @Override + protected Space replaceChild(Expression newChild) { + return new Space(location(), newChild); + } + + @Override + protected StringOperation operation() { + return StringOperation.SPACE; + } + + @Override + public DataType dataType() { + return DataType.KEYWORD; + } +} \ No newline at end of file diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionUtils.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionUtils.java new file mode 100644 index 0000000000000..059ad66583662 --- /dev/null +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionUtils.java @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.string; + +abstract class StringFunctionUtils { + + /** + * Trims the trailing whitespace characters from the given String. Uses {@link java.lang.Character.isWhitespace(char)} + * to determine if a character is whitespace or not. + * + * @param s the original String + * @return the resulting String + */ + static String trimTrailingWhitespaces(String s) { + if (!hasLength(s)) { + return s; + } + + StringBuilder sb = new StringBuilder(s); + while (sb.length() > 0 && Character.isWhitespace(sb.charAt(sb.length() - 1))) { + sb.deleteCharAt(sb.length() - 1); + } + return sb.toString(); + } + + /** + * Trims the leading whitespace characters from the given String. Uses {@link java.lang.Character.isWhitespace(char)} + * to determine if a character is whitespace or not. + * + * @param s the original String + * @return the resulting String + */ + static String trimLeadingWhitespaces(String s) { + if (!hasLength(s)) { + return s; + } + + StringBuilder sb = new StringBuilder(s); + while (sb.length() > 0 && Character.isWhitespace(sb.charAt(0))) { + sb.deleteCharAt(0); + } + return sb.toString(); + } + + private static boolean hasLength(String s) { + return (s != null && s.length() > 0); + } +} diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringProcessor.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringProcessor.java new file mode 100644 index 0000000000000..2a1ba3a10cf88 --- /dev/null +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringProcessor.java @@ -0,0 +1,160 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.string; + +import org.apache.lucene.util.UnicodeUtil; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.runtime.Processor; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Locale; +import java.util.function.Function; + +public class StringProcessor implements Processor { + + private interface StringFunction { + default R apply(Object o) { + if (!(o instanceof String || o instanceof Character)) { + throw new SqlIllegalArgumentException("A string/char is required; received [{}]", o); + } + + return doApply(o.toString()); + } + + R doApply(String s); + } + + private interface NumericFunction { + default R apply(Object o) { + if (!(o instanceof Number)) { + throw new SqlIllegalArgumentException("A number is required; received [{}]", o); + } + + return doApply((Number) o); + } + + R doApply(Number s); + } + + public enum StringOperation { + ASCII((String s) -> s.length() == 0 ? null : Integer.valueOf(s.charAt(0))), + CHAR((Number n) -> { + int i = n.intValue(); + return i < 0 || i > 255 ? null : String.valueOf((char) i); + }), + LCASE((String s) -> s.toLowerCase(Locale.ROOT)), + UCASE((String s) -> s.toUpperCase(Locale.ROOT)), + LENGTH((String s) -> StringFunctionUtils.trimTrailingWhitespaces(s).length()), + RTRIM((String s) -> StringFunctionUtils.trimTrailingWhitespaces(s)), + LTRIM((String s) -> StringFunctionUtils.trimLeadingWhitespaces(s)), + SPACE((Number n) -> { + int i = n.intValue(); + if (i < 0) { + return null; + }; + char[] spaces = new char[i]; + char whitespace = ' '; + Arrays.fill(spaces, whitespace); + + return new String(spaces); + }), + BIT_LENGTH((String s) -> UnicodeUtil.calcUTF16toUTF8Length(s, 0, s.length()) * 8), + CHAR_LENGTH(String::length); + + private final Function apply; + + StringOperation(StringFunction apply) { + this.apply = l -> l == null ? null : apply.apply(l); + } + + StringOperation(NumericFunction apply) { + this.apply = l -> l == null ? null : apply.apply((l)); + } + + StringOperation(Function apply) { + this(apply, false); + } + + /** + * Wrapper for nulls around the given function. + * If true, nulls are passed through, otherwise the function is short-circuited + * and null returned. + */ + StringOperation(Function apply, boolean nullAware) { + if (nullAware) { + this.apply = apply; + } else { + this.apply = l -> l == null ? null : apply.apply(l); + } + } + + public final Object apply(Object l) { + return apply.apply(l); + } + + /** + * "translate" the function name ("char") into a function name that is not a reserved keyword in java. + * Used in {@code InternalSqlScriptUtils#character(Number)}. + */ + @Override + public String toString() { + return this == CHAR ? "character" : super.toString(); + } + } + + public static final String NAME = "s"; + + private final StringOperation processor; + + public StringProcessor(StringOperation processor) { + this.processor = processor; + } + + public StringProcessor(StreamInput in) throws IOException { + processor = in.readEnum(StringOperation.class); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeEnum(processor); + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public Object process(Object input) { + return processor.apply(input); + } + + StringOperation processor() { + return processor; + } + + @Override + public boolean equals(Object obj) { + if (obj == null || obj.getClass() != getClass()) { + return false; + } + StringProcessor other = (StringProcessor) obj; + return processor == other.processor; + } + + @Override + public int hashCode() { + return processor.hashCode(); + } + + @Override + public String toString() { + return processor.toString(); + } +} \ No newline at end of file diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/UCase.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/UCase.java new file mode 100644 index 0000000000000..a030eeee7b97c --- /dev/null +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/UCase.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.string; + +import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.function.scalar.string.StringProcessor.StringOperation; +import org.elasticsearch.xpack.sql.tree.Location; +import org.elasticsearch.xpack.sql.tree.NodeInfo; +import org.elasticsearch.xpack.sql.type.DataType; + +/** + * Uppercases all lowercase letters in a string. + */ +public class UCase extends UnaryStringFunction { + + public UCase(Location location, Expression field) { + super(location, field); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, UCase::new, field()); + } + + @Override + protected UCase replaceChild(Expression newChild) { + return new UCase(location(), newChild); + } + + @Override + protected StringOperation operation() { + return StringOperation.UCASE; + } + + @Override + public DataType dataType() { + return DataType.KEYWORD; + } + +} diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/UnaryStringFunction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/UnaryStringFunction.java new file mode 100644 index 0000000000000..a0cfd50422cc0 --- /dev/null +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/UnaryStringFunction.java @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.string; + +import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.FieldAttribute; +import org.elasticsearch.xpack.sql.expression.function.scalar.UnaryScalarFunction; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinition; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinitions; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.UnaryProcessorDefinition; +import org.elasticsearch.xpack.sql.expression.function.scalar.script.ScriptTemplate; +import org.elasticsearch.xpack.sql.expression.function.scalar.string.StringProcessor.StringOperation; +import org.elasticsearch.xpack.sql.tree.Location; +import org.elasticsearch.xpack.sql.util.StringUtils; + +import java.util.Locale; +import java.util.Objects; + +import static java.lang.String.format; +import static org.elasticsearch.xpack.sql.expression.function.scalar.script.ParamsBuilder.paramsBuilder; + +public abstract class UnaryStringFunction extends UnaryScalarFunction { + + protected UnaryStringFunction(Location location, Expression field) { + super(location, field); + } + + @Override + public boolean foldable() { + return field().foldable(); + } + + @Override + public Object fold() { + return operation().apply(field().fold()); + } + + @Override + protected TypeResolution resolveType() { + if (!childrenResolved()) { + return new TypeResolution("Unresolved children"); + } + + return field().dataType().isString() ? TypeResolution.TYPE_RESOLVED : new TypeResolution( + "'%s' requires a string type, received %s", operation(), field().dataType().esType); + } + + @Override + protected final ProcessorDefinition makeProcessorDefinition() { + return new UnaryProcessorDefinition(location(), this, ProcessorDefinitions.toProcessorDefinition(field()), + new StringProcessor(operation())); + } + + protected abstract StringOperation operation(); + + @Override + protected ScriptTemplate asScriptFrom(FieldAttribute field) { + //TODO change this to use _source instead of the exact form (aka field.keyword for text fields) + return new ScriptTemplate(formatScript("doc[{}].value"), + paramsBuilder().variable(field.isInexact() ? field.exactAttribute().name() : field.name()).build(), + dataType()); + } + + @Override + protected String formatScript(String template) { + // basically, transform the script to InternalSqlScriptUtils.[function_name](other_function_or_field_name) + return super.formatScript( + format(Locale.ROOT, "{sql}.%s(%s)", + StringUtils.underscoreToLowerCamelCase(operation().toString()), + template)); + } + + @Override + public boolean equals(Object obj) { + if (obj == null || obj.getClass() != getClass()) { + return false; + } + UnaryStringFunction other = (UnaryStringFunction) obj; + return Objects.equals(other.field(), field()); + } + + @Override + public int hashCode() { + return Objects.hash(field()); + } +} \ No newline at end of file diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/UnaryStringIntFunction.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/UnaryStringIntFunction.java new file mode 100644 index 0000000000000..7e963eb9db78b --- /dev/null +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/UnaryStringIntFunction.java @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.string; + +import org.elasticsearch.xpack.sql.expression.Expression; +import org.elasticsearch.xpack.sql.expression.FieldAttribute; +import org.elasticsearch.xpack.sql.expression.function.scalar.UnaryScalarFunction; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinition; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.ProcessorDefinitions; +import org.elasticsearch.xpack.sql.expression.function.scalar.processor.definition.UnaryProcessorDefinition; +import org.elasticsearch.xpack.sql.expression.function.scalar.script.ScriptTemplate; +import org.elasticsearch.xpack.sql.expression.function.scalar.string.StringProcessor.StringOperation; +import org.elasticsearch.xpack.sql.tree.Location; +import org.elasticsearch.xpack.sql.util.StringUtils; + +import java.util.Locale; +import java.util.Objects; + +import static java.lang.String.format; +import static org.elasticsearch.xpack.sql.expression.function.scalar.script.ParamsBuilder.paramsBuilder; + +/** + * Base unary function for text manipulating SQL functions that receive as parameter a number + */ +public abstract class UnaryStringIntFunction extends UnaryScalarFunction { + + protected UnaryStringIntFunction(Location location, Expression field) { + super(location, field); + } + + @Override + public boolean foldable() { + return field().foldable(); + } + + @Override + public Object fold() { + return operation().apply(field().fold()); + } + + @Override + protected TypeResolution resolveType() { + if (!childrenResolved()) { + return new TypeResolution("Unresolved children"); + } + + return field().dataType().isInteger ? TypeResolution.TYPE_RESOLVED : new TypeResolution( + "'%s' requires a integer type, received %s", operation(), field().dataType().esType); + } + + @Override + protected final ProcessorDefinition makeProcessorDefinition() { + return new UnaryProcessorDefinition(location(), this, ProcessorDefinitions.toProcessorDefinition(field()), + new StringProcessor(operation())); + } + + protected abstract StringOperation operation(); + + @Override + protected ScriptTemplate asScriptFrom(FieldAttribute field) { + return new ScriptTemplate(formatScript("doc[{}].value"), + paramsBuilder().variable(field.name()).build(), + dataType()); + } + + @Override + protected String formatScript(String template) { + return super.formatScript( + format(Locale.ROOT, "{sql}.%s(%s)", + StringUtils.underscoreToLowerCamelCase(operation().toString()), + template)); + } + + @Override + public boolean equals(Object obj) { + if (obj == null || obj.getClass() != getClass()) { + return false; + } + UnaryStringIntFunction other = (UnaryStringIntFunction) obj; + return Objects.equals(other.field(), field()); + } + + @Override + public int hashCode() { + return Objects.hash(field()); + } +} \ No newline at end of file diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java index 802aa4a7c09bb..ccd5c24c6412d 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/expression/function/scalar/whitelist/InternalSqlScriptUtils.java @@ -6,6 +6,7 @@ package org.elasticsearch.xpack.sql.expression.function.scalar.whitelist; import org.elasticsearch.xpack.sql.expression.function.scalar.datetime.DateTimeFunction; +import org.elasticsearch.xpack.sql.expression.function.scalar.string.StringProcessor.StringOperation; /** * Whitelisted class for SQL scripts. @@ -19,4 +20,44 @@ private InternalSqlScriptUtils() {} public static Integer dateTimeChrono(long millis, String tzId, String chronoName) { return DateTimeFunction.dateTimeChrono(millis, tzId, chronoName); } + + public static Integer ascii(String s) { + return (Integer) StringOperation.ASCII.apply(s); + } + + public static Integer bitLength(String s) { + return (Integer) StringOperation.BIT_LENGTH.apply(s); + } + + public static String character(Number n) { + return (String) StringOperation.CHAR.apply(n); + } + + public static Integer charLength(String s) { + return (Integer) StringOperation.CHAR_LENGTH.apply(s); + } + + public static String lcase(String s) { + return (String) StringOperation.LCASE.apply(s); + } + + public static String ucase(String s) { + return (String) StringOperation.UCASE.apply(s); + } + + public static Integer length(String s) { + return (Integer) StringOperation.LENGTH.apply(s); + } + + public static String rtrim(String s) { + return (String) StringOperation.RTRIM.apply(s); + } + + public static String ltrim(String s) { + return (String) StringOperation.LTRIM.apply(s); + } + + public static String space(Number n) { + return (String) StringOperation.SPACE.apply(n); + } } diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java index dd0456e9aefc8..e691aef8d3e61 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/planner/QueryTranslator.java @@ -414,6 +414,9 @@ protected QueryTranslation asQuery(BinaryExpression e, boolean onAggs) { FieldAttribute fa = (FieldAttribute) e.left(); inexact = fa.isInexact(); target = nameOf(inexact ? fa : fa.exactAttribute()); + } else { + throw new SqlIllegalArgumentException("Scalar function ({}) not allowed (yet) as arguments for LIKE", + Expressions.name(e.left())); } if (e instanceof Like) { diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/util/StringUtils.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/util/StringUtils.java index e8bb9368d69ad..9570eaf1b6a06 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/util/StringUtils.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/util/StringUtils.java @@ -57,6 +57,33 @@ public static String camelCaseToUnderscore(String string) { } return sb.toString().toUpperCase(Locale.ROOT); } + + //CAMEL_CASE to camelCase + public static String underscoreToLowerCamelCase(String string) { + if (!Strings.hasText(string)) { + return EMPTY; + } + StringBuilder sb = new StringBuilder(); + String s = string.trim().toLowerCase(Locale.ROOT); + + boolean previousCharWasUnderscore = false; + for (int i = 0; i < s.length(); i++) { + char ch = s.charAt(i); + if (ch == '_') { + previousCharWasUnderscore = true; + } + else { + if (previousCharWasUnderscore) { + sb.append(Character.toUpperCase(ch)); + previousCharWasUnderscore = false; + } + else { + sb.append(ch); + } + } + } + return sb.toString(); + } public static String nullAsEmpty(String string) { return string == null ? EMPTY : string; diff --git a/x-pack/plugin/sql/src/main/resources/org/elasticsearch/xpack/sql/plugin/sql_whitelist.txt b/x-pack/plugin/sql/src/main/resources/org/elasticsearch/xpack/sql/plugin/sql_whitelist.txt index 8dae4f8c0d1d6..73a002c249f3c 100644 --- a/x-pack/plugin/sql/src/main/resources/org/elasticsearch/xpack/sql/plugin/sql_whitelist.txt +++ b/x-pack/plugin/sql/src/main/resources/org/elasticsearch/xpack/sql/plugin/sql_whitelist.txt @@ -9,4 +9,14 @@ class org.elasticsearch.xpack.sql.expression.function.scalar.whitelist.InternalSqlScriptUtils { Integer dateTimeChrono(long, String, String) + Integer ascii(String) + Integer bitLength(String) + String character(Number) + Integer charLength(String) + String lcase(String) + String ucase(String) + Integer length(String) + String rtrim(String) + String ltrim(String) + String space(Number) } \ No newline at end of file diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionProcessorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionProcessorTests.java new file mode 100644 index 0000000000000..dcfb8d278ff3f --- /dev/null +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionProcessorTests.java @@ -0,0 +1,160 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.sql.expression.function.scalar.string; + +import org.elasticsearch.common.io.stream.Writeable.Reader; +import org.elasticsearch.test.AbstractWireSerializingTestCase; +import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; +import org.elasticsearch.xpack.sql.expression.function.scalar.string.StringProcessor.StringOperation; + +import java.io.IOException; + +public class StringFunctionProcessorTests extends AbstractWireSerializingTestCase { + public static StringProcessor randomStringFunctionProcessor() { + return new StringProcessor(randomFrom(StringOperation.values())); + } + + @Override + protected StringProcessor createTestInstance() { + return randomStringFunctionProcessor(); + } + + @Override + protected Reader instanceReader() { + return StringProcessor::new; + } + + @Override + protected StringProcessor mutateInstance(StringProcessor instance) throws IOException { + return new StringProcessor(randomValueOtherThan(instance.processor(), () -> randomFrom(StringOperation.values()))); + } + + private void stringCharInputValidation(StringProcessor proc) { + SqlIllegalArgumentException siae = expectThrows(SqlIllegalArgumentException.class, () -> proc.process(123)); + assertEquals("A string/char is required; received [123]", siae.getMessage()); + } + + private void numericInputValidation(StringProcessor proc) { + SqlIllegalArgumentException siae = expectThrows(SqlIllegalArgumentException.class, () -> proc.process("A")); + assertEquals("A number is required; received [A]", siae.getMessage()); + } + + public void testAscii() { + StringProcessor proc = new StringProcessor(StringOperation.ASCII); + assertNull(proc.process(null)); + assertEquals(65, proc.process("A")); + // accepts chars as well + assertEquals(65, proc.process('A')); + assertEquals(65, proc.process("Alpha")); + // validate input + stringCharInputValidation(proc); + } + + public void testChar() { + StringProcessor proc = new StringProcessor(StringOperation.CHAR); + assertNull(proc.process(null)); + assertEquals("A", proc.process(65)); + assertNull(proc.process(256)); + assertNull(proc.process(-1)); + // validate input + numericInputValidation(proc); + } + + public void testLCase() { + StringProcessor proc = new StringProcessor(StringOperation.LCASE); + assertNull(proc.process(null)); + assertEquals("fulluppercase", proc.process("FULLUPPERCASE")); + assertEquals("someuppercase", proc.process("SomeUpPerCasE")); + assertEquals("fulllowercase", proc.process("fulllowercase")); + assertEquals("a", proc.process('A')); + + stringCharInputValidation(proc); + } + + public void testUCase() { + StringProcessor proc = new StringProcessor(StringOperation.UCASE); + assertNull(proc.process(null)); + assertEquals("FULLLOWERCASE", proc.process("fulllowercase")); + assertEquals("SOMELOWERCASE", proc.process("SomeLoweRCasE")); + assertEquals("FULLUPPERCASE", proc.process("FULLUPPERCASE")); + assertEquals("A", proc.process('a')); + + stringCharInputValidation(proc); + } + + public void testLength() { + StringProcessor proc = new StringProcessor(StringOperation.LENGTH); + assertNull(proc.process(null)); + assertEquals(7, proc.process("foo bar")); + assertEquals(0, proc.process("")); + assertEquals(0, proc.process(" ")); + assertEquals(7, proc.process("foo bar ")); + assertEquals(10, proc.process(" foo bar ")); + assertEquals(1, proc.process('f')); + + stringCharInputValidation(proc); + } + + public void testRTrim() { + StringProcessor proc = new StringProcessor(StringOperation.RTRIM); + assertNull(proc.process(null)); + assertEquals("foo bar", proc.process("foo bar")); + assertEquals("", proc.process("")); + assertEquals("", proc.process(" ")); + assertEquals("foo bar", proc.process("foo bar ")); + assertEquals(" foo bar", proc.process(" foo bar ")); + assertEquals("f", proc.process('f')); + + stringCharInputValidation(proc); + } + + public void testLTrim() { + StringProcessor proc = new StringProcessor(StringOperation.LTRIM); + assertNull(proc.process(null)); + assertEquals("foo bar", proc.process("foo bar")); + assertEquals("", proc.process("")); + assertEquals("", proc.process(" ")); + assertEquals("foo bar", proc.process(" foo bar")); + assertEquals("foo bar ", proc.process(" foo bar ")); + assertEquals("f", proc.process('f')); + + stringCharInputValidation(proc); + } + + public void testSpace() { + StringProcessor proc = new StringProcessor(StringOperation.SPACE); + int count = 7; + assertNull(proc.process(null)); + assertEquals(" ", proc.process(count)); + assertEquals(count, ((String) proc.process(count)).length()); + assertNotNull(proc.process(0)); + assertEquals("", proc.process(0)); + assertNull(proc.process(-1)); + + numericInputValidation(proc); + } + + public void testBitLength() { + StringProcessor proc = new StringProcessor(StringOperation.BIT_LENGTH); + assertNull(proc.process(null)); + assertEquals(56, proc.process("foo bar")); + assertEquals(0, proc.process("")); + assertEquals(8, proc.process('f')); + + stringCharInputValidation(proc); + } + + public void testCharLength() { + StringProcessor proc = new StringProcessor(StringOperation.CHAR_LENGTH); + assertNull(proc.process(null)); + assertEquals(7, proc.process("foo bar")); + assertEquals(0, proc.process("")); + assertEquals(1, proc.process('f')); + assertEquals(1, proc.process('€')); + + stringCharInputValidation(proc); + } +} diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java index 2a3d87b65c964..71f4dab679c99 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/planner/QueryTranslatorTests.java @@ -139,4 +139,14 @@ public void testDateRangeCast() { assertEquals("date", rq.field()); assertEquals(DateTime.parse("1969-05-13T12:34:56Z"), rq.lower()); } + + public void testLikeConstructsNotSupported() { + LogicalPlan p = plan("SELECT LTRIM(keyword) lt FROM test WHERE LTRIM(keyword) LIKE '%a%'"); + assertTrue(p instanceof Project); + p = ((Project) p).child(); + assertTrue(p instanceof Filter); + Expression condition = ((Filter) p).condition(); + SqlIllegalArgumentException ex = expectThrows(SqlIllegalArgumentException.class, () -> QueryTranslator.toQuery(condition, false)); + assertEquals("Scalar function (LTRIM(keyword)) not allowed (yet) as arguments for LIKE", ex.getMessage()); + } } \ No newline at end of file diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/ShowTestCase.java b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/ShowTestCase.java index 2605f6c27ce9b..32d1a67e5620f 100644 --- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/ShowTestCase.java +++ b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/ShowTestCase.java @@ -48,6 +48,9 @@ public void testShowFunctionsLikePrefix() throws IOException { assertThat(readLine(), containsString("----------")); assertThat(readLine(), RegexMatcher.matches("\\s*LOG\\s*\\|\\s*SCALAR\\s*")); assertThat(readLine(), RegexMatcher.matches("\\s*LOG10\\s*\\|\\s*SCALAR\\s*")); + assertThat(readLine(), RegexMatcher.matches("\\s*LCASE\\s*\\|\\s*SCALAR\\s*")); + assertThat(readLine(), RegexMatcher.matches("\\s*LENGTH\\s*\\|\\s*SCALAR\\s*")); + assertThat(readLine(), RegexMatcher.matches("\\s*LTRIM\\s*\\|\\s*SCALAR\\s*")); assertEquals("", readLine()); } diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/CsvSpecTestCase.java b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/CsvSpecTestCase.java index 99e8432370471..4aa599290e6e1 100644 --- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/CsvSpecTestCase.java +++ b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/CsvSpecTestCase.java @@ -37,6 +37,7 @@ public static List readScriptSpec() throws Exception { tests.addAll(readScriptSpec("/alias.csv-spec", parser)); tests.addAll(readScriptSpec("/nulls.csv-spec", parser)); tests.addAll(readScriptSpec("/nested.csv-spec", parser)); + tests.addAll(readScriptSpec("/functions.csv-spec", parser)); return tests; } diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/SqlSpecTestCase.java b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/SqlSpecTestCase.java index 3b5cae742d34b..b782e1474ea85 100644 --- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/SqlSpecTestCase.java +++ b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/SqlSpecTestCase.java @@ -34,6 +34,7 @@ public static List readScriptSpec() throws Exception { tests.addAll(readScriptSpec("/math.sql-spec", parser)); tests.addAll(readScriptSpec("/agg.sql-spec", parser)); tests.addAll(readScriptSpec("/arithmetic.sql-spec", parser)); + tests.addAll(readScriptSpec("/string-functions.sql-spec", parser)); return tests; } diff --git a/x-pack/qa/sql/src/main/resources/command.csv-spec b/x-pack/qa/sql/src/main/resources/command.csv-spec index d54fb6bf1554c..47615a4c3ae84 100644 --- a/x-pack/qa/sql/src/main/resources/command.csv-spec +++ b/x-pack/qa/sql/src/main/resources/command.csv-spec @@ -69,6 +69,16 @@ SIN |SCALAR SINH |SCALAR SQRT |SCALAR TAN |SCALAR +ASCII |SCALAR +CHAR |SCALAR +BIT_LENGTH |SCALAR +CHAR_LENGTH |SCALAR +LCASE |SCALAR +LENGTH |SCALAR +LTRIM |SCALAR +RTRIM |SCALAR +SPACE |SCALAR +UCASE |SCALAR SCORE |SCORE ; @@ -90,6 +100,7 @@ ACOS |SCALAR ASIN |SCALAR ATAN |SCALAR ATAN2 |SCALAR +ASCII |SCALAR ; showFunctionsWithPatternChar diff --git a/x-pack/qa/sql/src/main/resources/docs.csv-spec b/x-pack/qa/sql/src/main/resources/docs.csv-spec index 54509b21df36f..8d7debee331fc 100644 --- a/x-pack/qa/sql/src/main/resources/docs.csv-spec +++ b/x-pack/qa/sql/src/main/resources/docs.csv-spec @@ -222,6 +222,16 @@ SIN |SCALAR SINH |SCALAR SQRT |SCALAR TAN |SCALAR +ASCII |SCALAR +CHAR |SCALAR +BIT_LENGTH |SCALAR +CHAR_LENGTH |SCALAR +LCASE |SCALAR +LENGTH |SCALAR +LTRIM |SCALAR +RTRIM |SCALAR +SPACE |SCALAR +UCASE |SCALAR SCORE |SCORE // end::showFunctions @@ -249,7 +259,8 @@ ABS |SCALAR ACOS |SCALAR ASIN |SCALAR ATAN |SCALAR -ATAN2 |SCALAR +ATAN2 |SCALAR +ASCII |SCALAR // end::showFunctionsLikeWildcard ; diff --git a/x-pack/qa/sql/src/main/resources/functions.csv-spec b/x-pack/qa/sql/src/main/resources/functions.csv-spec new file mode 100644 index 0000000000000..09320c3d384c0 --- /dev/null +++ b/x-pack/qa/sql/src/main/resources/functions.csv-spec @@ -0,0 +1,30 @@ +bitLengthGroupByAndOrderBy +SELECT BIT_LENGTH(first_name), COUNT(*) count FROM "test_emp" GROUP BY BIT_LENGTH(first_name) ORDER BY BIT_LENGTH(first_name) LIMIT 10; + +BIT_LENGTH(first_name):i| count:l +24 |4 +32 |11 +40 |16 +48 |24 +56 |19 +64 |14 +72 |10 +80 |1 +88 |1 +; + +bitLengthOrderByFieldWithWhere +SELECT BIT_LENGTH(first_name) len, first_name FROM "test_emp" WHERE BIT_LENGTH(first_name) > 64 ORDER BY first_name LIMIT 10; + +len:i | first_name:s +80 |Adamantios +72 |Alejandro +72 |Alejandro +72 |Chirstian +72 |Cristinel +72 |Duangkaew +72 |Eberhardt +72 |Margareta +72 |Prasadram +88 |Sreekrishna +; diff --git a/x-pack/qa/sql/src/main/resources/string-functions.sql-spec b/x-pack/qa/sql/src/main/resources/string-functions.sql-spec new file mode 100644 index 0000000000000..d9a35edf1b030 --- /dev/null +++ b/x-pack/qa/sql/src/main/resources/string-functions.sql-spec @@ -0,0 +1,76 @@ +stringAscii +SELECT ASCII(first_name) s FROM "test_emp" WHERE emp_no < 10010 ORDER BY emp_no; +stringChar +SELECT CHAR(emp_no % 10000) m, first_name FROM "test_emp" WHERE emp_no < 10010 ORDER BY emp_no; + +stringAsciiFilter +SELECT emp_no, ASCII(first_name) a FROM "test_emp" WHERE ASCII(first_name) < 10010 ORDER BY emp_no; + +stringAsciiEqualsConstant +SELECT emp_no, ASCII(first_name) a, first_name name FROM "test_emp" WHERE ASCII(first_name) = 65 ORDER BY emp_no; + +//https://github.com/elastic/elasticsearch/issues/31863 +//stringSelectConstantAsciiEqualsConstant +//SELECT ASCII('A') = 65 a FROM "test_emp" WHERE ASCII('A') = 65 ORDER BY emp_no; + +stringCharFilter +SELECT emp_no, CHAR(emp_no % 10000) m FROM "test_emp" WHERE CHAR(emp_no % 10000) = 'A'; + +lcaseFilter +SELECT LCASE(first_name) lc, CHAR(ASCII(LCASE(first_name))) chr FROM "test_emp" WHERE CHAR(ASCII(LCASE(first_name))) = 'a'; + +ltrimFilter +SELECT LTRIM(first_name) lt FROM "test_emp" WHERE LTRIM(first_name) = 'Bob'; + +//Unsupported yet +//ltrimFilterWithLike +//SELECT LTRIM("first_name") lt FROM "test_emp" WHERE LTRIM("first_name") LIKE '%a%'; + +rtrimFilter +SELECT RTRIM(first_name) rt FROM "test_emp" WHERE RTRIM(first_name) = 'Johnny'; + +spaceFilter +SELECT SPACE(languages) spaces, languages FROM "test_emp" WHERE SPACE(languages) = ' '; + +spaceFilterWithLengthFunctions +SELECT SPACE(languages) spaces, languages, first_name FROM "test_emp" WHERE CHAR_LENGTH(SPACE(languages)) = 3 ORDER BY first_name; + +ucaseFilter +SELECT UCASE(gender) uppercased, COUNT(*) count FROM "test_emp" WHERE UCASE(gender) = 'F' GROUP BY UCASE(gender); + +// +// Group and order by +// +asciiGroupByAndOrderBy +SELECT ASCII(first_name) A, COUNT(*) count FROM "test_emp" WHERE ASCII(first_name) < 75 GROUP BY ASCII(first_name) ORDER BY ASCII(first_name) DESC; + +charGroupByAndOrderBy +SELECT CHAR(emp_no % 10000) C FROM "test_emp" WHERE emp_no > 10010 GROUP BY CHAR(emp_no % 10000) ORDER BY CHAR(emp_no % 10000) DESC LIMIT 20; + +//this would fail because H2 returns the result of char_length as Long, while we use a DataType of type String (size Integer.MAX_VALUE) and we return an Integer +//CAST is used as an "workaround" +charLengthGroupByAndHavingAndOrderBy +SELECT CAST(CHAR_LENGTH("first_name") AS INT) cl, COUNT(*) count FROM "test_emp" GROUP BY "first_name" HAVING COUNT(*)>1 ORDER BY CHAR_LENGTH("first_name") ; + +//this one, without ORDER BY, would return different results than H2. In ES, the default ordering of the composite aggregation +//values is "asc" while in H2 there is no default ordering +lcaseGroupByAndOrderBy +SELECT LCASE(first_name) lc, CHAR(ASCII(LCASE(first_name))) chr FROM "test_emp" GROUP BY LCASE(first_name) ORDER BY LCASE(first_name); + +ucaseGroupByAndOrderBy +SELECT UCASE(gender) uc, COUNT(*) count FROM "test_emp" GROUP BY UCASE(gender) ORDER BY UCASE(gender) DESC; + +rtrimGroupByAndOrderBy +SELECT RTRIM(first_name) rt FROM "test_emp" GROUP BY RTRIM(first_name) HAVING COUNT(*)>1; + +ltrimGroupByAndOrderBy +SELECT LTRIM(first_name) lt FROM "test_emp" GROUP BY LTRIM(first_name) HAVING COUNT(*)>1; + +spaceGroupByWithCharLength +SELECT CAST(CHAR_LENGTH(SPACE(languages)) AS INT) cls FROM "test_emp" GROUP BY CHAR_LENGTH(SPACE(languages)); + +spaceGroupByAndOrderBy +SELECT SPACE("languages") s, COUNT(*) count FROM "test_emp" GROUP BY SPACE("languages") ORDER BY SPACE(languages); + +spaceGroupByAndOrderByWithCharLength +SELECT SPACE("languages") s, COUNT(*) count, CAST(CHAR_LENGTH(SPACE("languages")) AS INT) cls FROM "test_emp" WHERE "languages" IS NOT NULL GROUP BY SPACE("languages") ORDER BY SPACE("languages"); From efe4bde04f2f1b4921ba01433f678f49fc096fdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 12 Jul 2018 15:50:58 +0200 Subject: [PATCH 42/50] Add Expected Reciprocal Rank metric (#31891) This change adds Expected Reciprocal Rank (ERR) as a ranking evaluation metric as descriped in: Chapelle, O., Metlzer, D., Zhang, Y., & Grinspan, P. (2009). Expected reciprocal rank for graded relevance. Proceeding of the 18th ACM Conference on Information and Knowledge Management. https://doi.org/10.1145/1645953.1646033 ERR is an extension of the classical reciprocal rank to the graded relevance case and assumes a cascade browsing model. It quantifies the usefulness of a document at rank `i` conditioned on the degree of relevance of the items at ranks less than `i`. ERR seems to be gain traction as an alternative to (n)DCG, so it seems like a good metric to support. Also ERR seems to be the default optimization metric used for training in RankLib, a widely used learning to rank library. Relates to #29653 --- .../xcontent/ConstructingObjectParser.java | 1 + .../rankeval/DiscountedCumulativeGain.java | 6 +- .../rankeval/ExpectedReciprocalRank.java | 295 ++++++++++++++++++ .../DiscountedCumulativeGainTests.java | 12 +- .../rankeval/ExpectedReciprocalRankTests.java | 217 +++++++++++++ 5 files changed, 522 insertions(+), 9 deletions(-) create mode 100644 modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRank.java create mode 100644 modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRankTests.java diff --git a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ConstructingObjectParser.java b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ConstructingObjectParser.java index d61bd8a5dbbdb..e880781fad781 100644 --- a/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ConstructingObjectParser.java +++ b/libs/x-content/src/main/java/org/elasticsearch/common/xcontent/ConstructingObjectParser.java @@ -294,6 +294,7 @@ public void declareNamedObjects(BiConsumer> consumer, NamedOb } } + @Override public String getName() { return objectParser.getName(); } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java index 01a6e35299b29..cab3237732301 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java @@ -326,9 +326,9 @@ public boolean equals(Object obj) { return false; } DiscountedCumulativeGain.Detail other = (DiscountedCumulativeGain.Detail) obj; - return (this.dcg == other.dcg && - this.idcg == other.idcg && - this.unratedDocs == other.unratedDocs); + return Double.compare(this.dcg, other.dcg) == 0 && + Double.compare(this.idcg, other.idcg) == 0 && + this.unratedDocs == other.unratedDocs; } @Override diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRank.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRank.java new file mode 100644 index 0000000000000..4aac29f299d67 --- /dev/null +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRank.java @@ -0,0 +1,295 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.search.SearchHit; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; +import static org.elasticsearch.index.rankeval.EvaluationMetric.joinHitsWithRatings; + +/** + * Implementation of the Expected Reciprocal Rank metric described in:

    + * + * Chapelle, O., Metlzer, D., Zhang, Y., & Grinspan, P. (2009).
    + * Expected reciprocal rank for graded relevance.
    + * Proceeding of the 18th ACM Conference on Information and Knowledge Management - CIKM ’09, 621.
    + * https://doi.org/10.1145/1645953.1646033 + */ +public class ExpectedReciprocalRank implements EvaluationMetric { + + /** the default search window size */ + private static final int DEFAULT_K = 10; + + /** the search window size */ + private final int k; + + /** + * Optional. If set, this will be the rating for docs that are unrated in the ranking evaluation request + */ + private final Integer unknownDocRating; + + private final int maxRelevance; + + private final double two_pow_maxRelevance; + + public static final String NAME = "expected_reciprocal_rank"; + + public ExpectedReciprocalRank(int maxRelevance) { + this(maxRelevance, null, DEFAULT_K); + } + + /** + * @param maxRelevance + * the maximal relevance judgment in the evaluation dataset + * @param unknownDocRating + * the rating for documents the user hasn't supplied an explicit + * rating for. Can be {@code null}, in which case document is + * skipped. + * @param k + * the search window size all request use. + */ + public ExpectedReciprocalRank(int maxRelevance, @Nullable Integer unknownDocRating, int k) { + this.maxRelevance = maxRelevance; + this.unknownDocRating = unknownDocRating; + this.k = k; + // we can pre-calculate the constant used in metric calculation + this.two_pow_maxRelevance = Math.pow(2, this.maxRelevance); + } + + ExpectedReciprocalRank(StreamInput in) throws IOException { + this.maxRelevance = in.readVInt(); + this.unknownDocRating = in.readOptionalVInt(); + this.k = in.readVInt(); + this.two_pow_maxRelevance = Math.pow(2, this.maxRelevance); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(maxRelevance); + out.writeOptionalVInt(unknownDocRating); + out.writeVInt(k); + } + + @Override + public String getWriteableName() { + return NAME; + } + + int getK() { + return this.k; + } + + int getMaxRelevance() { + return this.maxRelevance; + } + + /** + * get the rating used for unrated documents + */ + public Integer getUnknownDocRating() { + return this.unknownDocRating; + } + + + @Override + public Optional forcedSearchSize() { + return Optional.of(k); + } + + @Override + public EvalQueryQuality evaluate(String taskId, SearchHit[] hits, List ratedDocs) { + List ratedHits = joinHitsWithRatings(hits, ratedDocs); + if (ratedHits.size() > this.k) { + ratedHits = ratedHits.subList(0, k); + } + List ratingsInSearchHits = new ArrayList<>(ratedHits.size()); + int unratedResults = 0; + for (RatedSearchHit hit : ratedHits) { + // unknownDocRating might be null, in which case unrated will be ignored in the calculation. + // we still need to add them as a placeholder so the rank of the subsequent ratings is correct + ratingsInSearchHits.add(hit.getRating().orElse(unknownDocRating)); + if (hit.getRating().isPresent() == false) { + unratedResults++; + } + } + + double p = 1; + double err = 0; + int rank = 1; + for (Integer rating : ratingsInSearchHits) { + if (rating != null) { + double probR = probabilityOfRelevance(rating); + err = err + (p * probR / rank); + p = p * (1 - probR); + } + rank++; + } + + EvalQueryQuality evalQueryQuality = new EvalQueryQuality(taskId, err); + evalQueryQuality.addHitsAndRatings(ratedHits); + evalQueryQuality.setMetricDetails(new Detail(unratedResults)); + return evalQueryQuality; + } + + double probabilityOfRelevance(Integer rating) { + return (Math.pow(2, rating) - 1) / this.two_pow_maxRelevance; + } + + private static final ParseField K_FIELD = new ParseField("k"); + private static final ParseField UNKNOWN_DOC_RATING_FIELD = new ParseField("unknown_doc_rating"); + private static final ParseField MAX_RELEVANCE_FIELD = new ParseField("maximum_relevance"); + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("dcg", false, + args -> { + int maxRelevance = (Integer) args[0]; + Integer optK = (Integer) args[2]; + return new ExpectedReciprocalRank(maxRelevance, (Integer) args[1], + optK == null ? DEFAULT_K : optK); + }); + + + static { + PARSER.declareInt(constructorArg(), MAX_RELEVANCE_FIELD); + PARSER.declareInt(optionalConstructorArg(), UNKNOWN_DOC_RATING_FIELD); + PARSER.declareInt(optionalConstructorArg(), K_FIELD); + } + + public static ExpectedReciprocalRank fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.startObject(NAME); + builder.field(MAX_RELEVANCE_FIELD.getPreferredName(), this.maxRelevance); + if (unknownDocRating != null) { + builder.field(UNKNOWN_DOC_RATING_FIELD.getPreferredName(), this.unknownDocRating); + } + builder.field(K_FIELD.getPreferredName(), this.k); + builder.endObject(); + builder.endObject(); + return builder; + } + + @Override + public final boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + ExpectedReciprocalRank other = (ExpectedReciprocalRank) obj; + return this.k == other.k && + this.maxRelevance == other.maxRelevance + && Objects.equals(unknownDocRating, other.unknownDocRating); + } + + @Override + public final int hashCode() { + return Objects.hash(unknownDocRating, k, maxRelevance); + } + + public static final class Detail implements MetricDetail { + + private static ParseField UNRATED_FIELD = new ParseField("unrated_docs"); + private final int unratedDocs; + + Detail(int unratedDocs) { + this.unratedDocs = unratedDocs; + } + + Detail(StreamInput in) throws IOException { + this.unratedDocs = in.readVInt(); + } + + @Override + public + String getMetricName() { + return NAME; + } + + @Override + public XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException { + return builder.field(UNRATED_FIELD.getPreferredName(), this.unratedDocs); + } + + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>(NAME, true, args -> { + return new Detail((Integer) args[0]); + }); + + static { + PARSER.declareInt(constructorArg(), UNRATED_FIELD); + } + + public static Detail fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(this.unratedDocs); + } + + @Override + public String getWriteableName() { + return NAME; + } + + /** + * @return the number of unrated documents in the search results + */ + public Object getUnratedDocs() { + return this.unratedDocs; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + ExpectedReciprocalRank.Detail other = (ExpectedReciprocalRank.Detail) obj; + return this.unratedDocs == other.unratedDocs; + } + + @Override + public int hashCode() { + return Objects.hash(this.unratedDocs); + } + } +} + diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java index 24ac600a11398..56b0c692c411a 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java @@ -55,7 +55,7 @@ public class DiscountedCumulativeGainTests extends ESTestCase { /** * Assuming the docs are ranked in the following order: * - * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / log_2(rank + 1) + * rank | relevance | 2^(relevance) - 1 | log_2(rank + 1) | (2^(relevance) - 1) / log_2(rank + 1) * ------------------------------------------------------------------------------------------- * 1 | 3 | 7.0 | 1.0 | 7.0 | 7.0 |  * 2 | 2 | 3.0 | 1.5849625007211563 | 1.8927892607143721 @@ -82,7 +82,7 @@ public void testDCGAt() { * Check with normalization: to get the maximal possible dcg, sort documents by * relevance in descending order * - * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / log_2(rank + 1) + * rank | relevance | 2^(relevance) - 1 | log_2(rank + 1) | (2^(relevance) - 1) / log_2(rank + 1) * --------------------------------------------------------------------------------------- * 1 | 3 | 7.0 | 1.0  | 7.0 * 2 | 3 | 7.0 | 1.5849625007211563 | 4.416508275000202 @@ -101,7 +101,7 @@ public void testDCGAt() { * This tests metric when some documents in the search result don't have a * rating provided by the user. * - * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / log_2(rank + 1) + * rank | relevance | 2^(relevance) - 1 | log_2(rank + 1) | (2^(relevance) - 1) / log_2(rank + 1) * ------------------------------------------------------------------------------------------- * 1 | 3 | 7.0 | 1.0 | 7.0 2 |  * 2 | 3.0 | 1.5849625007211563 | 1.8927892607143721 @@ -134,7 +134,7 @@ public void testDCGAtSixMissingRatings() { * Check with normalization: to get the maximal possible dcg, sort documents by * relevance in descending order * - * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / log_2(rank + 1) + * rank | relevance | 2^(relevance) - 1 | log_2(rank + 1) | (2^(relevance) - 1) / log_2(rank + 1) * ---------------------------------------------------------------------------------------- * 1 | 3 | 7.0 | 1.0  | 7.0 * 2 | 3 | 7.0 | 1.5849625007211563 | 4.416508275000202 @@ -154,7 +154,7 @@ public void testDCGAtSixMissingRatings() { * documents than search hits because we restrict DCG to be calculated at the * fourth position * - * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / log_2(rank + 1) + * rank | relevance | 2^(relevance) - 1 | log_2(rank + 1) | (2^(relevance) - 1) / log_2(rank + 1) * ------------------------------------------------------------------------------------------- * 1 | 3 | 7.0 | 1.0 | 7.0 2 |  * 2 | 3.0 | 1.5849625007211563 | 1.8927892607143721 @@ -191,7 +191,7 @@ public void testDCGAtFourMoreRatings() { * Check with normalization: to get the maximal possible dcg, sort documents by * relevance in descending order * - * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / log_2(rank + 1) + * rank | relevance | 2^(relevance) - 1 | log_2(rank + 1) | (2^(relevance) - 1) / log_2(rank + 1) * --------------------------------------------------------------------------------------- * 1 | 3 | 7.0 | 1.0  | 7.0 * 2 | 3 | 7.0 | 1.5849625007211563 | 4.416508275000202 diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRankTests.java new file mode 100644 index 0000000000000..a888bf010f9e2 --- /dev/null +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ExpectedReciprocalRankTests.java @@ -0,0 +1,217 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.text.Text; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentParseException; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.index.Index; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchShardTarget; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode; +import static org.elasticsearch.test.XContentTestUtils.insertRandomFields; +import static org.hamcrest.CoreMatchers.containsString; + +public class ExpectedReciprocalRankTests extends ESTestCase { + + private static final double DELTA = 10E-14; + + public void testProbabilityOfRelevance() { + ExpectedReciprocalRank err = new ExpectedReciprocalRank(5); + assertEquals(0.0, err.probabilityOfRelevance(0), 0.0); + assertEquals(1d/32d, err.probabilityOfRelevance(1), 0.0); + assertEquals(3d/32d, err.probabilityOfRelevance(2), 0.0); + assertEquals(7d/32d, err.probabilityOfRelevance(3), 0.0); + assertEquals(15d/32d, err.probabilityOfRelevance(4), 0.0); + assertEquals(31d/32d, err.probabilityOfRelevance(5), 0.0); + } + + /** + * Assuming the result ranking is + * + * 

    {@code
+     * rank | relevance | probR / r | p        | p * probR / r
+     * -------------------------------------------------------
+     * 1    | 3         | 0.875     | 1        | 0.875       |
+     * 2    | 2         | 0.1875    | 0.125    | 0.0234375   |
+     * 3    | 0         | 0         | 0.078125 | 0           |
+     * 4    | 1         | 0.03125   | 0.078125 | 0.00244140625 |
+     * }
    + * + * err => sum of last column + */ + public void testERRAt() { + List rated = new ArrayList<>(); + Integer[] relevanceRatings = new Integer[] { 3, 2, 0, 1}; + SearchHit[] hits = createSearchHits(rated, relevanceRatings); + ExpectedReciprocalRank err = new ExpectedReciprocalRank(3, 0, 3); + assertEquals(0.8984375, err.evaluate("id", hits, rated).getQualityLevel(), DELTA); + // take 4th rank into window + err = new ExpectedReciprocalRank(3, 0, 4); + assertEquals(0.8984375 + 0.00244140625, err.evaluate("id", hits, rated).getQualityLevel(), DELTA); + } + + /** + * Assuming the result ranking is + * + * 
    {@code
+     * rank | relevance | probR / r | p        | p * probR / r
+     * -------------------------------------------------------
+     * 1    | 3         | 0.875     | 1        | 0.875       |
+     * 2    | n/a       | n/a       | 0.125    | n/a   |
+     * 3    | 0         | 0         | 0.125    | 0           |
+     * 4    | 1         | 0.03125   | 0.125    | 0.00390625 |
+     * }
    + * + * err => sum of last column + */ + public void testERRMissingRatings() { + List rated = new ArrayList<>(); + Integer[] relevanceRatings = new Integer[] { 3, null, 0, 1}; + SearchHit[] hits = createSearchHits(rated, relevanceRatings); + ExpectedReciprocalRank err = new ExpectedReciprocalRank(3, null, 4); + EvalQueryQuality evaluation = err.evaluate("id", hits, rated); + assertEquals(0.875 + 0.00390625, evaluation.getQualityLevel(), DELTA); + assertEquals(1, ((ExpectedReciprocalRank.Detail) evaluation.getMetricDetails()).getUnratedDocs()); + // if we supply e.g. 2 as unknown docs rating, it should be the same as in the other test above + err = new ExpectedReciprocalRank(3, 2, 4); + assertEquals(0.8984375 + 0.00244140625, err.evaluate("id", hits, rated).getQualityLevel(), DELTA); + } + + private SearchHit[] createSearchHits(List rated, Integer[] relevanceRatings) { + SearchHit[] hits = new SearchHit[relevanceRatings.length]; + for (int i = 0; i < relevanceRatings.length; i++) { + if (relevanceRatings[i] != null) { + rated.add(new RatedDocument("index", Integer.toString(i), relevanceRatings[i])); + } + hits[i] = new SearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); + hits[i].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0, null)); + } + return hits; + } + + /** + * test that metric returns 0.0 when there are no search results + */ + public void testNoResults() throws Exception { + ExpectedReciprocalRank err = new ExpectedReciprocalRank(5, 0, 10); + assertEquals(0.0, err.evaluate("id", new SearchHit[0], Collections.emptyList()).getQualityLevel(), DELTA); + } + + public void testParseFromXContent() throws IOException { + assertParsedCorrect("{ \"unknown_doc_rating\": 2, \"maximum_relevance\": 5, \"k\" : 15 }", 2, 5, 15); + assertParsedCorrect("{ \"unknown_doc_rating\": 2, \"maximum_relevance\": 4 }", 2, 4, 10); + assertParsedCorrect("{ \"maximum_relevance\": 4, \"k\": 23 }", null, 4, 23); + } + + private void assertParsedCorrect(String xContent, Integer expectedUnknownDocRating, int expectedMaxRelevance, int expectedK) + throws IOException { + try (XContentParser parser = createParser(JsonXContent.jsonXContent, xContent)) { + ExpectedReciprocalRank errAt = ExpectedReciprocalRank.fromXContent(parser); + assertEquals(expectedUnknownDocRating, errAt.getUnknownDocRating()); + assertEquals(expectedK, errAt.getK()); + assertEquals(expectedMaxRelevance, errAt.getMaxRelevance()); + } + } + + public static ExpectedReciprocalRank createTestItem() { + Integer unknownDocRating = frequently() ? Integer.valueOf(randomIntBetween(0, 10)) : null; + int maxRelevance = randomIntBetween(1, 10); + return new ExpectedReciprocalRank(maxRelevance, unknownDocRating, randomIntBetween(1, 10)); + } + + public void testXContentRoundtrip() throws IOException { + ExpectedReciprocalRank testItem = createTestItem(); + XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS)); + try (XContentParser itemParser = createParser(shuffled)) { + itemParser.nextToken(); + itemParser.nextToken(); + ExpectedReciprocalRank parsedItem = ExpectedReciprocalRank.fromXContent(itemParser); + assertNotSame(testItem, parsedItem); + assertEquals(testItem, parsedItem); + assertEquals(testItem.hashCode(), parsedItem.hashCode()); + } + } + + public void testXContentParsingIsNotLenient() throws IOException { + ExpectedReciprocalRank testItem = createTestItem(); + XContentType xContentType = randomFrom(XContentType.values()); + BytesReference originalBytes = toShuffledXContent(testItem, xContentType, ToXContent.EMPTY_PARAMS, randomBoolean()); + BytesReference withRandomFields = insertRandomFields(xContentType, originalBytes, null, random()); + try (XContentParser parser = createParser(xContentType.xContent(), withRandomFields)) { + parser.nextToken(); + parser.nextToken(); + XContentParseException exception = expectThrows(XContentParseException.class, + () -> DiscountedCumulativeGain.fromXContent(parser)); + assertThat(exception.getMessage(), containsString("[dcg] unknown field")); + } + } + + public void testMetricDetails() { + int unratedDocs = randomIntBetween(0, 100); + ExpectedReciprocalRank.Detail detail = new ExpectedReciprocalRank.Detail(unratedDocs); + assertEquals(unratedDocs, detail.getUnratedDocs()); + } + + public void testSerialization() throws IOException { + ExpectedReciprocalRank original = createTestItem(); + ExpectedReciprocalRank deserialized = ESTestCase.copyWriteable(original, new NamedWriteableRegistry(Collections.emptyList()), + ExpectedReciprocalRank::new); + assertEquals(deserialized, original); + assertEquals(deserialized.hashCode(), original.hashCode()); + assertNotSame(deserialized, original); + } + + public void testEqualsAndHash() throws IOException { + checkEqualsAndHashCode(createTestItem(), original -> { + return new ExpectedReciprocalRank(original.getMaxRelevance(), original.getUnknownDocRating(), original.getK()); + }, ExpectedReciprocalRankTests::mutateTestItem); + } + + private static ExpectedReciprocalRank mutateTestItem(ExpectedReciprocalRank original) { + switch (randomIntBetween(0, 2)) { + case 0: + return new ExpectedReciprocalRank(original.getMaxRelevance() + 1, original.getUnknownDocRating(), original.getK()); + case 1: + return new ExpectedReciprocalRank(original.getMaxRelevance(), + randomValueOtherThan(original.getUnknownDocRating(), () -> randomIntBetween(0, 10)), original.getK()); + case 2: + return new ExpectedReciprocalRank(original.getMaxRelevance(), original.getUnknownDocRating(), + randomValueOtherThan(original.getK(), () -> randomIntBetween(1, 10))); + default: + throw new IllegalArgumentException("mutation variant not allowed"); + } + } +} From 02fa0d45aeb0b535cc06afca3c8cf5e039444454 Mon Sep 17 00:00:00 2001 From: Dimitris Athanasiou Date: Thu, 12 Jul 2018 15:36:37 +0100 Subject: [PATCH 43/50] [6.x][ML] Ensure immutability of MlMetadata (#31994) The test failure in #31916 revealed that updating rules on a job was modifying the detectors list in-place. That meant the old cluster state and the updated cluster state had no difference and thus the change was not propagated to non-master nodes. This commit fixes that and also reviews all of ML metadata in order to ensure immutability. Closes #31916 --- .../core/ml/datafeed/DatafeedConfig.java | 16 +-- .../core/ml/datafeed/DatafeedUpdate.java | 12 ++ .../core/ml/job/config/AnalysisConfig.java | 30 ++-- .../xpack/core/ml/job/config/Detector.java | 4 +- .../xpack/core/ml/job/config/Job.java | 15 +- .../xpack/core/ml/job/config/JobUpdate.java | 60 ++++++-- .../xpack/core/ml/job/config/RuleScope.java | 2 +- .../core/ml/datafeed/DatafeedUpdateTests.java | 30 +++- .../core/ml/job/config/JobUpdateTests.java | 136 ++++++++++++++---- 9 files changed, 237 insertions(+), 68 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java index d06c911e13cfe..e9ac704171bef 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedConfig.java @@ -156,14 +156,14 @@ private DatafeedConfig(String id, String jobId, TimeValue queryDelay, TimeValue this.jobId = jobId; this.queryDelay = queryDelay; this.frequency = frequency; - this.indices = indices; - this.types = types; + this.indices = indices == null ? null : Collections.unmodifiableList(indices); + this.types = types == null ? null : Collections.unmodifiableList(types); this.query = query; this.aggregations = aggregations; - this.scriptFields = scriptFields; + this.scriptFields = scriptFields == null ? null : Collections.unmodifiableList(scriptFields); this.scrollSize = scrollSize; this.chunkingConfig = chunkingConfig; - this.headers = Objects.requireNonNull(headers); + this.headers = Collections.unmodifiableMap(headers); } public DatafeedConfig(StreamInput in) throws IOException { @@ -172,19 +172,19 @@ public DatafeedConfig(StreamInput in) throws IOException { this.queryDelay = in.readOptionalTimeValue(); this.frequency = in.readOptionalTimeValue(); if (in.readBoolean()) { - this.indices = in.readList(StreamInput::readString); + this.indices = Collections.unmodifiableList(in.readList(StreamInput::readString)); } else { this.indices = null; } if (in.readBoolean()) { - this.types = in.readList(StreamInput::readString); + this.types = Collections.unmodifiableList(in.readList(StreamInput::readString)); } else { this.types = null; } this.query = in.readNamedWriteable(QueryBuilder.class); this.aggregations = in.readOptionalWriteable(AggregatorFactories.Builder::new); if (in.readBoolean()) { - this.scriptFields = in.readList(SearchSourceBuilder.ScriptField::new); + this.scriptFields = Collections.unmodifiableList(in.readList(SearchSourceBuilder.ScriptField::new)); } else { this.scriptFields = null; } @@ -195,7 +195,7 @@ public DatafeedConfig(StreamInput in) throws IOException { } this.chunkingConfig = in.readOptionalWriteable(ChunkingConfig::new); if (in.getVersion().onOrAfter(Version.V_6_2_0)) { - this.headers = in.readMap(StreamInput::readString, StreamInput::readString); + this.headers = Collections.unmodifiableMap(in.readMap(StreamInput::readString, StreamInput::readString)); } else { this.headers = Collections.emptyMap(); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdate.java index 27498bd1549ee..5d8fd3ffc71e9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdate.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdate.java @@ -352,6 +352,18 @@ public String toString() { return Strings.toString(this); } + boolean isNoop(DatafeedConfig datafeed) { + return (frequency == null || Objects.equals(frequency, datafeed.getFrequency())) + && (queryDelay == null || Objects.equals(queryDelay, datafeed.getQueryDelay())) + && (indices == null || Objects.equals(indices, datafeed.getIndices())) + && (types == null || Objects.equals(types, datafeed.getTypes())) + && (query == null || Objects.equals(query, datafeed.getQuery())) + && (scrollSize == null || Objects.equals(scrollSize, datafeed.getQueryDelay())) + && (aggregations == null || Objects.equals(aggregations, datafeed.getAggregations())) + && (scriptFields == null || Objects.equals(scriptFields, datafeed.getScriptFields())) + && (chunkingConfig == null || Objects.equals(chunkingConfig, datafeed.getChunkingConfig())); + } + public static class Builder { private String id; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfig.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfig.java index 02d8b6f529327..0c702e5afb0bb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfig.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/AnalysisConfig.java @@ -144,20 +144,20 @@ private AnalysisConfig(TimeValue bucketSpan, String categorizationFieldName, Lis this.latency = latency; this.categorizationFieldName = categorizationFieldName; this.categorizationAnalyzerConfig = categorizationAnalyzerConfig; - this.categorizationFilters = categorizationFilters; + this.categorizationFilters = categorizationFilters == null ? null : Collections.unmodifiableList(categorizationFilters); this.summaryCountFieldName = summaryCountFieldName; - this.influencers = influencers; + this.influencers = Collections.unmodifiableList(influencers); this.overlappingBuckets = overlappingBuckets; this.resultFinalizationWindow = resultFinalizationWindow; this.multivariateByFields = multivariateByFields; - this.multipleBucketSpans = multipleBucketSpans; + this.multipleBucketSpans = multipleBucketSpans == null ? null : Collections.unmodifiableList(multipleBucketSpans); this.usePerPartitionNormalization = usePerPartitionNormalization; } public AnalysisConfig(StreamInput in) throws IOException { bucketSpan = in.readTimeValue(); categorizationFieldName = in.readOptionalString(); - categorizationFilters = in.readBoolean() ? in.readList(StreamInput::readString) : null; + categorizationFilters = in.readBoolean() ? Collections.unmodifiableList(in.readList(StreamInput::readString)) : null; if (in.getVersion().onOrAfter(Version.V_6_2_0)) { categorizationAnalyzerConfig = in.readOptionalWriteable(CategorizationAnalyzerConfig::new); } else { @@ -165,8 +165,8 @@ public AnalysisConfig(StreamInput in) throws IOException { } latency = in.readOptionalTimeValue(); summaryCountFieldName = in.readOptionalString(); - detectors = in.readList(Detector::new); - influencers = in.readList(StreamInput::readString); + detectors = Collections.unmodifiableList(in.readList(Detector::new)); + influencers = Collections.unmodifiableList(in.readList(StreamInput::readString)); overlappingBuckets = in.readOptionalBoolean(); resultFinalizationWindow = in.readOptionalLong(); multivariateByFields = in.readOptionalBoolean(); @@ -176,7 +176,7 @@ public AnalysisConfig(StreamInput in) throws IOException { for (int i = 0; i < arraySize; i++) { spans.add(in.readTimeValue()); } - multipleBucketSpans = spans; + multipleBucketSpans = Collections.unmodifiableList(spans); } else { multipleBucketSpans = null; } @@ -487,18 +487,20 @@ public Builder(List detectors) { } public Builder(AnalysisConfig analysisConfig) { - this.detectors = analysisConfig.detectors; + this.detectors = new ArrayList<>(analysisConfig.detectors); this.bucketSpan = analysisConfig.bucketSpan; this.latency = analysisConfig.latency; this.categorizationFieldName = analysisConfig.categorizationFieldName; - this.categorizationFilters = analysisConfig.categorizationFilters; + this.categorizationFilters = analysisConfig.categorizationFilters == null ? null + : new ArrayList<>(analysisConfig.categorizationFilters); this.categorizationAnalyzerConfig = analysisConfig.categorizationAnalyzerConfig; this.summaryCountFieldName = analysisConfig.summaryCountFieldName; - this.influencers = analysisConfig.influencers; + this.influencers = new ArrayList<>(analysisConfig.influencers); this.overlappingBuckets = analysisConfig.overlappingBuckets; this.resultFinalizationWindow = analysisConfig.resultFinalizationWindow; this.multivariateByFields = analysisConfig.multivariateByFields; - this.multipleBucketSpans = analysisConfig.multipleBucketSpans; + this.multipleBucketSpans = analysisConfig.multipleBucketSpans == null ? null + : new ArrayList<>(analysisConfig.multipleBucketSpans); this.usePerPartitionNormalization = analysisConfig.usePerPartitionNormalization; } @@ -518,6 +520,10 @@ public void setDetectors(List detectors) { this.detectors = sequentialIndexDetectors; } + public void setDetector(int detectorIndex, Detector detector) { + detectors.set(detectorIndex, detector); + } + public void setBucketSpan(TimeValue bucketSpan) { this.bucketSpan = bucketSpan; } @@ -543,7 +549,7 @@ public void setSummaryCountFieldName(String summaryCountFieldName) { } public void setInfluencers(List influencers) { - this.influencers = influencers; + this.influencers = ExceptionsHelper.requireNonNull(influencers, INFLUENCERS.getPreferredName()); } public void setOverlappingBuckets(Boolean overlappingBuckets) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Detector.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Detector.java index 8a0213f291d5d..ec6cbc305afd0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Detector.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Detector.java @@ -253,7 +253,7 @@ public Detector(StreamInput in) throws IOException { useNull = in.readBoolean(); excludeFrequent = in.readBoolean() ? ExcludeFrequent.readFromStream(in) : null; if (in.getVersion().onOrAfter(DetectionRule.VERSION_INTRODUCED)) { - rules = in.readList(DetectionRule::new); + rules = Collections.unmodifiableList(in.readList(DetectionRule::new)); } else { in.readList(DetectionRule::readOldFormat); rules = Collections.emptyList(); @@ -513,7 +513,7 @@ public Builder(Detector detector) { partitionFieldName = detector.partitionFieldName; useNull = detector.useNull; excludeFrequent = detector.excludeFrequent; - rules = new ArrayList<>(detector.getRules()); + rules = new ArrayList<>(detector.rules); detectorIndex = detector.detectorIndex; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java index a5ecae34d888a..f3c68542fa4f4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/Job.java @@ -190,7 +190,7 @@ private Job(String jobId, String jobType, Version jobVersion, List group this.jobId = jobId; this.jobType = jobType; this.jobVersion = jobVersion; - this.groups = groups; + this.groups = Collections.unmodifiableList(groups); this.description = description; this.createTime = createTime; this.finishedTime = finishedTime; @@ -204,7 +204,7 @@ private Job(String jobId, String jobType, Version jobVersion, List group this.backgroundPersistInterval = backgroundPersistInterval; this.modelSnapshotRetentionDays = modelSnapshotRetentionDays; this.resultsRetentionDays = resultsRetentionDays; - this.customSettings = customSettings; + this.customSettings = customSettings == null ? null : Collections.unmodifiableMap(customSettings); this.modelSnapshotId = modelSnapshotId; this.resultsIndexName = resultsIndexName; this.deleted = deleted; @@ -219,7 +219,7 @@ public Job(StreamInput in) throws IOException { jobVersion = null; } if (in.getVersion().onOrAfter(Version.V_6_1_0)) { - groups = in.readList(StreamInput::readString); + groups = Collections.unmodifiableList(in.readList(StreamInput::readString)); } else { groups = Collections.emptyList(); } @@ -240,7 +240,8 @@ public Job(StreamInput in) throws IOException { backgroundPersistInterval = in.readOptionalTimeValue(); modelSnapshotRetentionDays = in.readOptionalLong(); resultsRetentionDays = in.readOptionalLong(); - customSettings = in.readMap(); + Map readCustomSettings = in.readMap(); + customSettings = readCustomSettings == null ? null : Collections.unmodifiableMap(readCustomSettings); modelSnapshotId = in.readOptionalString(); resultsIndexName = in.readString(); deleted = in.readBoolean(); @@ -603,7 +604,8 @@ public boolean equals(Object other) { && Objects.equals(this.lastDataTime, that.lastDataTime) && Objects.equals(this.establishedModelMemory, that.establishedModelMemory) && Objects.equals(this.analysisConfig, that.analysisConfig) - && Objects.equals(this.analysisLimits, that.analysisLimits) && Objects.equals(this.dataDescription, that.dataDescription) + && Objects.equals(this.analysisLimits, that.analysisLimits) + && Objects.equals(this.dataDescription, that.dataDescription) && Objects.equals(this.modelPlotConfig, that.modelPlotConfig) && Objects.equals(this.renormalizationWindowDays, that.renormalizationWindowDays) && Objects.equals(this.backgroundPersistInterval, that.backgroundPersistInterval) @@ -1002,6 +1004,7 @@ public boolean equals(Object o) { return Objects.equals(this.id, that.id) && Objects.equals(this.jobType, that.jobType) && Objects.equals(this.jobVersion, that.jobVersion) + && Objects.equals(this.groups, that.groups) && Objects.equals(this.description, that.description) && Objects.equals(this.analysisConfig, that.analysisConfig) && Objects.equals(this.analysisLimits, that.analysisLimits) @@ -1023,7 +1026,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(id, jobType, jobVersion, description, analysisConfig, analysisLimits, dataDescription, createTime, + return Objects.hash(id, jobType, jobVersion, groups, description, analysisConfig, analysisLimits, dataDescription, createTime, finishedTime, lastDataTime, establishedModelMemory, modelPlotConfig, renormalizationWindowDays, backgroundPersistInterval, modelSnapshotRetentionDays, resultsRetentionDays, customSettings, modelSnapshotId, resultsIndexName, deleted); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java index 5c4e313bace08..be59599b8aec2 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java @@ -346,6 +346,9 @@ public Set getUpdateFields() { */ public Job mergeWithJob(Job source, ByteSizeValue maxModelMemoryLimit) { Job.Builder builder = new Job.Builder(source); + AnalysisConfig currentAnalysisConfig = source.getAnalysisConfig(); + AnalysisConfig.Builder newAnalysisConfig = new AnalysisConfig.Builder(currentAnalysisConfig); + if (groups != null) { builder.setGroups(groups); } @@ -353,26 +356,23 @@ public Job mergeWithJob(Job source, ByteSizeValue maxModelMemoryLimit) { builder.setDescription(description); } if (detectorUpdates != null && detectorUpdates.isEmpty() == false) { - AnalysisConfig ac = source.getAnalysisConfig(); - int numDetectors = ac.getDetectors().size(); + int numDetectors = currentAnalysisConfig.getDetectors().size(); for (DetectorUpdate dd : detectorUpdates) { if (dd.getDetectorIndex() >= numDetectors) { throw ExceptionsHelper.badRequestException("Supplied detector_index [{}] is >= the number of detectors [{}]", dd.getDetectorIndex(), numDetectors); } - Detector.Builder detectorbuilder = new Detector.Builder(ac.getDetectors().get(dd.getDetectorIndex())); + Detector.Builder detectorBuilder = new Detector.Builder(currentAnalysisConfig.getDetectors().get(dd.getDetectorIndex())); if (dd.getDescription() != null) { - detectorbuilder.setDetectorDescription(dd.getDescription()); + detectorBuilder.setDetectorDescription(dd.getDescription()); } if (dd.getRules() != null) { - detectorbuilder.setRules(dd.getRules()); + detectorBuilder.setRules(dd.getRules()); } - ac.getDetectors().set(dd.getDetectorIndex(), detectorbuilder.build()); - } - AnalysisConfig.Builder acBuilder = new AnalysisConfig.Builder(ac); - builder.setAnalysisConfig(acBuilder); + newAnalysisConfig.setDetector(dd.getDetectorIndex(), detectorBuilder.build()); + } } if (modelPlotConfig != null) { builder.setModelPlotConfig(modelPlotConfig); @@ -395,9 +395,7 @@ public Job mergeWithJob(Job source, ByteSizeValue maxModelMemoryLimit) { builder.setResultsRetentionDays(resultsRetentionDays); } if (categorizationFilters != null) { - AnalysisConfig.Builder analysisConfigBuilder = new AnalysisConfig.Builder(source.getAnalysisConfig()); - analysisConfigBuilder.setCategorizationFilters(categorizationFilters); - builder.setAnalysisConfig(analysisConfigBuilder); + newAnalysisConfig.setCategorizationFilters(categorizationFilters); } if (customSettings != null) { builder.setCustomSettings(customSettings); @@ -416,9 +414,47 @@ public Job mergeWithJob(Job source, ByteSizeValue maxModelMemoryLimit) { if (jobVersion != null) { builder.setJobVersion(jobVersion); } + + builder.setAnalysisConfig(newAnalysisConfig); return builder.build(); } + boolean isNoop(Job job) { + return (groups == null || Objects.equals(groups, job.getGroups())) + && (description == null || Objects.equals(description, job.getDescription())) + && (modelPlotConfig == null || Objects.equals(modelPlotConfig, job.getModelPlotConfig())) + && (analysisLimits == null || Objects.equals(analysisLimits, job.getAnalysisLimits())) + && updatesDetectors(job) == false + && (renormalizationWindowDays == null || Objects.equals(renormalizationWindowDays, job.getRenormalizationWindowDays())) + && (backgroundPersistInterval == null || Objects.equals(backgroundPersistInterval, job.getBackgroundPersistInterval())) + && (modelSnapshotRetentionDays == null || Objects.equals(modelSnapshotRetentionDays, job.getModelSnapshotRetentionDays())) + && (resultsRetentionDays == null || Objects.equals(resultsRetentionDays, job.getResultsRetentionDays())) + && (categorizationFilters == null + || Objects.equals(categorizationFilters, job.getAnalysisConfig().getCategorizationFilters())) + && (customSettings == null || Objects.equals(customSettings, job.getCustomSettings())) + && (modelSnapshotId == null || Objects.equals(modelSnapshotId, job.getModelSnapshotId())) + && (establishedModelMemory == null || Objects.equals(establishedModelMemory, job.getEstablishedModelMemory())) + && (jobVersion == null || Objects.equals(jobVersion, job.getJobVersion())); + } + + boolean updatesDetectors(Job job) { + AnalysisConfig analysisConfig = job.getAnalysisConfig(); + if (detectorUpdates == null) { + return false; + } + for (DetectorUpdate detectorUpdate : detectorUpdates) { + if (detectorUpdate.description == null && detectorUpdate.rules == null) { + continue; + } + Detector detector = analysisConfig.getDetectors().get(detectorUpdate.detectorIndex); + if (Objects.equals(detectorUpdate.description, detector.getDetectorDescription()) == false + || Objects.equals(detectorUpdate.rules, detector.getRules()) == false) { + return true; + } + } + return false; + } + @Override public boolean equals(Object other) { if (this == other) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleScope.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleScope.java index b6b3b4e061bdd..0b11fa0e15b3d 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleScope.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/RuleScope.java @@ -60,7 +60,7 @@ public RuleScope() { } public RuleScope(Map scope) { - this.scope = Objects.requireNonNull(scope); + this.scope = Collections.unmodifiableMap(scope); } public RuleScope(StreamInput in) throws IOException { diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdateTests.java index 358f9d1c97bd7..7e0615e85f8c9 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdateTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/datafeed/DatafeedUpdateTests.java @@ -5,6 +5,7 @@ */ package org.elasticsearch.xpack.core.ml.datafeed; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Settings; @@ -24,6 +25,7 @@ import org.elasticsearch.search.builder.SearchSourceBuilder.ScriptField; import org.elasticsearch.test.AbstractSerializingTestCase; import org.elasticsearch.xpack.core.ml.datafeed.ChunkingConfig.Mode; +import org.elasticsearch.xpack.core.ml.job.config.JobTests; import java.util.ArrayList; import java.util.Collections; @@ -31,6 +33,7 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; public class DatafeedUpdateTests extends AbstractSerializingTestCase { @@ -40,8 +43,12 @@ protected DatafeedUpdate createTestInstance() { } public static DatafeedUpdate createRandomized(String datafeedId) { + return createRandomized(datafeedId, null); + } + + public static DatafeedUpdate createRandomized(String datafeedId, @Nullable DatafeedConfig datafeed) { DatafeedUpdate.Builder builder = new DatafeedUpdate.Builder(datafeedId); - if (randomBoolean()) { + if (randomBoolean() && datafeed == null) { builder.setJobId(randomAlphaOfLength(10)); } if (randomBoolean()) { @@ -68,7 +75,7 @@ public static DatafeedUpdate createRandomized(String datafeedId) { } builder.setScriptFields(scriptFields); } - if (randomBoolean()) { + if (randomBoolean() && datafeed == null) { // can only test with a single agg as the xcontent order gets randomized by test base class and then // the actual xcontent isn't the same and test fail. // Testing with a single agg is ok as we don't have special list writeable / xconent logic @@ -184,6 +191,25 @@ public void testApply_givenAggregations() { AggregationBuilders.histogram("a").interval(300000).field("time").subAggregation(maxTime)))); } + public void testApply_GivenRandomUpdates_AssertImmutability() { + for (int i = 0; i < 100; ++i) { + DatafeedConfig datafeed = DatafeedConfigTests.createRandomizedDatafeedConfig(JobTests.randomValidJobId()); + if (datafeed.getAggregations() != null) { + DatafeedConfig.Builder withoutAggs = new DatafeedConfig.Builder(datafeed); + withoutAggs.setAggregations(null); + datafeed = withoutAggs.build(); + } + DatafeedUpdate update = createRandomized(datafeed.getId(), datafeed); + while (update.isNoop(datafeed)) { + update = createRandomized(datafeed.getId(), datafeed); + } + + DatafeedConfig updatedDatafeed = update.apply(datafeed, Collections.emptyMap()); + + assertThat(datafeed, not(equalTo(updatedDatafeed))); + } + } + @Override protected DatafeedUpdate mutateInstance(DatafeedUpdate instance) { DatafeedUpdate.Builder builder = new DatafeedUpdate.Builder(instance); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java index 780c1a02d841e..75b47a0c43615 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java @@ -7,6 +7,7 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.Version; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.unit.ByteSizeUnit; import org.elasticsearch.common.unit.ByteSizeValue; @@ -18,10 +19,13 @@ import java.util.Arrays; import java.util.Collections; import java.util.Date; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; import static org.mockito.Mockito.mock; public class JobUpdateTests extends AbstractSerializingTestCase { @@ -30,7 +34,15 @@ public class JobUpdateTests extends AbstractSerializingTestCase { @Override protected JobUpdate createTestInstance() { - JobUpdate.Builder update = new JobUpdate.Builder(randomAlphaOfLength(4)); + return createRandom(randomAlphaOfLength(4), null); + } + + /** + * Creates a completely random update when the job is null + * or a random update that is is valid for the given job + */ + public JobUpdate createRandom(String jobId, @Nullable Job job) { + JobUpdate.Builder update = new JobUpdate.Builder(jobId); if (randomBoolean()) { int groupsNum = randomIntBetween(0, 10); List groups = new ArrayList<>(groupsNum); @@ -43,28 +55,16 @@ protected JobUpdate createTestInstance() { update.setDescription(randomAlphaOfLength(20)); } if (randomBoolean()) { - int size = randomInt(10); - List detectorUpdates = new ArrayList<>(size); - for (int i = 0; i < size; i++) { - String detectorDescription = null; - if (randomBoolean()) { - detectorDescription = randomAlphaOfLength(12); - } - List detectionRules = null; - if (randomBoolean()) { - detectionRules = new ArrayList<>(); - detectionRules.add(new DetectionRule.Builder( - Collections.singletonList(new RuleCondition(RuleCondition.AppliesTo.ACTUAL, Operator.GT, 5))).build()); - } - detectorUpdates.add(new JobUpdate.DetectorUpdate(i, detectorDescription, detectionRules)); - } + List detectorUpdates = job == null ? createRandomDetectorUpdates() + : createRandomDetectorUpdatesForJob(job); update.setDetectorUpdates(detectorUpdates); } if (randomBoolean()) { update.setModelPlotConfig(new ModelPlotConfig(randomBoolean(), randomAlphaOfLength(10))); } if (randomBoolean()) { - update.setAnalysisLimits(AnalysisLimitsTests.createRandomized()); + update.setAnalysisLimits(AnalysisLimits.validateAndSetDefaults(AnalysisLimitsTests.createRandomized(), null, + AnalysisLimits.DEFAULT_MODEL_MEMORY_LIMIT_MB)); } if (randomBoolean()) { update.setRenormalizationWindowDays(randomNonNegativeLong()); @@ -78,7 +78,7 @@ protected JobUpdate createTestInstance() { if (randomBoolean()) { update.setResultsRetentionDays(randomNonNegativeLong()); } - if (randomBoolean()) { + if (randomBoolean() && jobSupportsCategorizationFilters(job)) { update.setCategorizationFilters(Arrays.asList(generateRandomStringArray(10, 10, false))); } if (randomBoolean()) { @@ -97,6 +97,77 @@ protected JobUpdate createTestInstance() { return update.build(); } + private static boolean jobSupportsCategorizationFilters(@Nullable Job job) { + if (job == null) { + return true; + } + if (job.getAnalysisConfig().getCategorizationFieldName() == null) { + return false; + } + if (job.getAnalysisConfig().getCategorizationAnalyzerConfig() != null) { + return false; + } + return true; + } + + private static List createRandomDetectorUpdates() { + int size = randomInt(10); + List detectorUpdates = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + String detectorDescription = null; + if (randomBoolean()) { + detectorDescription = randomAlphaOfLength(12); + } + List detectionRules = null; + if (randomBoolean()) { + detectionRules = new ArrayList<>(); + detectionRules.add(new DetectionRule.Builder( + Collections.singletonList(new RuleCondition(RuleCondition.AppliesTo.ACTUAL, Operator.GT, 5))).build()); + } + detectorUpdates.add(new JobUpdate.DetectorUpdate(i, detectorDescription, detectionRules)); + } + return detectorUpdates; + } + + private static List createRandomDetectorUpdatesForJob(Job job) { + AnalysisConfig analysisConfig = job.getAnalysisConfig(); + int size = randomInt(analysisConfig.getDetectors().size()); + List detectorUpdates = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + String detectorDescription = null; + if (randomBoolean()) { + detectorDescription = randomAlphaOfLength(12); + } + int rulesSize = randomBoolean() ? randomIntBetween(1, 5) : 0; + List detectionRules = rulesSize == 0 ? null : new ArrayList<>(rulesSize); + for (int ruleIndex = 0; ruleIndex < rulesSize; ++ruleIndex) { + int detectorIndex = randomInt(analysisConfig.getDetectors().size() - 1); + Detector detector = analysisConfig.getDetectors().get(detectorIndex); + List analysisFields = detector.extractAnalysisFields(); + if (randomBoolean() || analysisFields.isEmpty()) { + detectionRules.add(new DetectionRule.Builder(Collections.singletonList(new RuleCondition( + randomFrom(RuleCondition.AppliesTo.values()), randomFrom(Operator.values()), randomDouble()))).build()); + } else { + RuleScope.Builder ruleScope = RuleScope.builder(); + int scopeSize = randomIntBetween(1, analysisFields.size()); + Set analysisFieldsPickPot = new HashSet<>(analysisFields); + for (int scopeIndex = 0; scopeIndex < scopeSize; ++scopeIndex) { + String scopedField = randomFrom(analysisFieldsPickPot); + analysisFieldsPickPot.remove(scopedField); + if (randomBoolean()) { + ruleScope.include(scopedField, MlFilterTests.randomValidFilterId()); + } else { + ruleScope.exclude(scopedField, MlFilterTests.randomValidFilterId()); + } + } + detectionRules.add(new DetectionRule.Builder(ruleScope).build()); + } + } + detectorUpdates.add(new JobUpdate.DetectorUpdate(i, detectorDescription, detectionRules)); + } + return detectorUpdates; + } + @Override protected Writeable.Reader instanceReader() { return JobUpdate::new; @@ -153,8 +224,9 @@ public void testMergeWithJob() { jobBuilder.setAnalysisConfig(ac); jobBuilder.setDataDescription(new DataDescription.Builder()); jobBuilder.setCreateTime(new Date()); + Job job = jobBuilder.build(); - Job updatedJob = update.mergeWithJob(jobBuilder.build(), new ByteSizeValue(0L)); + Job updatedJob = update.mergeWithJob(job, new ByteSizeValue(0L)); assertEquals(update.getGroups(), updatedJob.getGroups()); assertEquals(update.getDescription(), updatedJob.getDescription()); @@ -169,12 +241,26 @@ public void testMergeWithJob() { assertEquals(update.getModelSnapshotId(), updatedJob.getModelSnapshotId()); assertEquals(update.getJobVersion(), updatedJob.getJobVersion()); for (JobUpdate.DetectorUpdate detectorUpdate : update.getDetectorUpdates()) { - assertNotNull(updatedJob.getAnalysisConfig().getDetectors().get(detectorUpdate.getDetectorIndex()).getDetectorDescription()); - assertEquals(detectorUpdate.getDescription(), - updatedJob.getAnalysisConfig().getDetectors().get(detectorUpdate.getDetectorIndex()).getDetectorDescription()); - assertNotNull(updatedJob.getAnalysisConfig().getDetectors().get(detectorUpdate.getDetectorIndex()).getDetectorDescription()); - assertEquals(detectorUpdate.getRules(), - updatedJob.getAnalysisConfig().getDetectors().get(detectorUpdate.getDetectorIndex()).getRules()); + Detector updatedDetector = updatedJob.getAnalysisConfig().getDetectors().get(detectorUpdate.getDetectorIndex()); + assertNotNull(updatedDetector); + assertEquals(detectorUpdate.getDescription(), updatedDetector.getDetectorDescription()); + assertEquals(detectorUpdate.getRules(), updatedDetector.getRules()); + } + + assertThat(job, not(equalTo(updatedJob))); + } + + public void testMergeWithJob_GivenRandomUpdates_AssertImmutability() { + for (int i = 0; i < 100; ++i) { + Job job = JobTests.createRandomizedJob(); + JobUpdate update = createRandom(job.getId(), job); + while (update.isNoop(job)) { + update = createRandom(job.getId(), job); + } + + Job updatedJob = update.mergeWithJob(job, new ByteSizeValue(0L)); + + assertThat(job, not(equalTo(updatedJob))); } } From 9a0a0eba1d1eafe22e4523410d1ebd2ca2b85916 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 12 Jul 2018 17:38:30 +0200 Subject: [PATCH 44/50] Fix unreachable error condition in AmazonS3Fixture (#32005) The `else` branch where currently the error response should be thrown is not reachable because `handler` is always non-null inside the previous outer check. Moving error creation into an else branch on the other condition check, removing the other superflous check for non-null handler inside the first branch. --- .../repositories/s3/AmazonS3Fixture.java | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java index e21f2bf71496b..7dab265c1ea64 100644 --- a/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java +++ b/plugins/repository-s3/src/test/java/org/elasticsearch/repositories/s3/AmazonS3Fixture.java @@ -18,14 +18,15 @@ */ package org.elasticsearch.repositories.s3; -import org.elasticsearch.test.fixture.AbstractHttpFixture; import com.amazonaws.util.DateUtils; + import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.path.PathTrie; import org.elasticsearch.common.util.concurrent.ConcurrentCollections; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.RestUtils; +import org.elasticsearch.test.fixture.AbstractHttpFixture; import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; @@ -93,23 +94,21 @@ protected Response handle(final Request request) throws IOException { return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad access key", ""); } - if (handler != null) { - final String bucket = request.getParam("bucket"); - if (bucket != null && permittedBucket.equals(bucket) == false) { - // allow a null bucket to support the multi-object-delete API which - // passes the bucket name in the host header instead of the URL. - if (buckets.containsKey(bucket)) { - return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad bucket", ""); - } else { - return newBucketNotFoundError(request.getId(), bucket); - } + final String bucket = request.getParam("bucket"); + if (bucket != null && permittedBucket.equals(bucket) == false) { + // allow a null bucket to support the multi-object-delete API which + // passes the bucket name in the host header instead of the URL. + if (buckets.containsKey(bucket)) { + return newError(request.getId(), RestStatus.FORBIDDEN, "AccessDenied", "Bad bucket", ""); + } else { + return newBucketNotFoundError(request.getId(), bucket); } - return handler.handle(request); - } else { - return newInternalError(request.getId(), "No handler defined for request [" + request + "]"); } + return handler.handle(request); + + } else { + return newInternalError(request.getId(), "No handler defined for request [" + request + "]"); } - return null; } public static void main(final String[] args) throws Exception { From 8df23532d82f05ea75be628edafa726b6267e154 Mon Sep 17 00:00:00 2001 From: Tim Brooks Date: Thu, 12 Jul 2018 09:56:12 -0600 Subject: [PATCH 45/50] Add Get Snapshots High Level REST API (#31980) With this commit we add the get snapshots API to the Java high level REST client. --- .../client/RequestConverters.java | 21 +++ .../elasticsearch/client/SnapshotClient.java | 31 +++++ .../client/RequestConvertersTests.java | 61 +++++++++ .../org/elasticsearch/client/SnapshotIT.java | 38 ++++++ .../SnapshotClientDocumentationIT.java | 87 +++++++++++++ .../snapshot/get_snapshots.asciidoc | 108 ++++++++++++++++ .../high-level/supported-apis.asciidoc | 2 + .../snapshots/get/GetSnapshotsResponse.java | 30 +++++ .../elasticsearch/snapshots/SnapshotInfo.java | 60 +++------ .../snapshots/SnapshotShardFailure.java | 120 ++++++++++-------- .../get/GetSnapshotsResponseTests.java | 62 +++++++++ 11 files changed, 524 insertions(+), 96 deletions(-) create mode 100644 docs/java-rest/high-level/snapshot/get_snapshots.asciidoc create mode 100644 server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java index 9396baa9420d9..de2ac130f79e1 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java @@ -39,6 +39,7 @@ import org.elasticsearch.action.admin.cluster.settings.ClusterGetSettingsRequest; import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest; +import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest; import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest; import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest; @@ -953,6 +954,26 @@ static Request createSnapshot(CreateSnapshotRequest createSnapshotRequest) throw return request; } + static Request getSnapshots(GetSnapshotsRequest getSnapshotsRequest) { + EndpointBuilder endpointBuilder = new EndpointBuilder().addPathPartAsIs("_snapshot") + .addPathPart(getSnapshotsRequest.repository()); + String endpoint; + if (getSnapshotsRequest.snapshots().length == 0) { + endpoint = endpointBuilder.addPathPart("_all").build(); + } else { + endpoint = endpointBuilder.addCommaSeparatedPathParts(getSnapshotsRequest.snapshots()).build(); + } + + Request request = new Request(HttpGet.METHOD_NAME, endpoint); + + Params parameters = new Params(request); + parameters.withMasterTimeout(getSnapshotsRequest.masterNodeTimeout()); + parameters.putParam("ignore_unavailable", Boolean.toString(getSnapshotsRequest.ignoreUnavailable())); + parameters.putParam("verbose", Boolean.toString(getSnapshotsRequest.verbose())); + + return request; + } + static Request deleteSnapshot(DeleteSnapshotRequest deleteSnapshotRequest) { String endpoint = new EndpointBuilder().addPathPartAsIs("_snapshot") .addPathPart(deleteSnapshotRequest.repository()) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java index 4482fce2edf94..fa147a338de0a 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/SnapshotClient.java @@ -32,6 +32,8 @@ import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest; import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotResponse; +import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; +import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; import java.io.IOException; @@ -190,6 +192,35 @@ public void createSnapshotAsync(CreateSnapshotRequest createSnapshotRequest, Req CreateSnapshotResponse::fromXContent, listener, emptySet()); } + /** + * Get snapshots. + * See Snapshot and Restore + * API on elastic.co + * + * @param getSnapshotsRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public GetSnapshotsResponse get(GetSnapshotsRequest getSnapshotsRequest, RequestOptions options) throws IOException { + return restHighLevelClient.performRequestAndParseEntity(getSnapshotsRequest, RequestConverters::getSnapshots, options, + GetSnapshotsResponse::fromXContent, emptySet()); + } + + /** + * Asynchronously get snapshots. + * See Snapshot and Restore + * API on elastic.co + * + * @param getSnapshotsRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public void getAsync(GetSnapshotsRequest getSnapshotsRequest, RequestOptions options, ActionListener listener) { + restHighLevelClient.performRequestAsyncAndParseEntity(getSnapshotsRequest, RequestConverters::getSnapshots, options, + GetSnapshotsResponse::fromXContent, listener, emptySet()); + } + /** * Deletes a snapshot. * See Snapshot and Restore diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java index f18c102217bfe..89753bda82e8d 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java @@ -40,6 +40,7 @@ import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest; import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest; +import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; import org.elasticsearch.action.admin.cluster.storedscripts.DeleteStoredScriptRequest; import org.elasticsearch.action.admin.cluster.storedscripts.GetStoredScriptRequest; import org.elasticsearch.action.admin.indices.alias.Alias; @@ -2143,6 +2144,66 @@ public void testCreateSnapshot() throws IOException { assertToXContentBody(createSnapshotRequest, request.getEntity()); } + public void testGetSnapshots() { + Map expectedParams = new HashMap<>(); + String repository = randomIndicesNames(1, 1)[0]; + String snapshot1 = "snapshot1-" + randomAlphaOfLengthBetween(2, 5).toLowerCase(Locale.ROOT); + String snapshot2 = "snapshot2-" + randomAlphaOfLengthBetween(2, 5).toLowerCase(Locale.ROOT); + + String endpoint = String.format(Locale.ROOT, "/_snapshot/%s/%s,%s", repository, snapshot1, snapshot2); + + GetSnapshotsRequest getSnapshotsRequest = new GetSnapshotsRequest(); + getSnapshotsRequest.repository(repository); + getSnapshotsRequest.snapshots(Arrays.asList(snapshot1, snapshot2).toArray(new String[0])); + setRandomMasterTimeout(getSnapshotsRequest, expectedParams); + + if (randomBoolean()) { + boolean ignoreUnavailable = randomBoolean(); + getSnapshotsRequest.ignoreUnavailable(ignoreUnavailable); + expectedParams.put("ignore_unavailable", Boolean.toString(ignoreUnavailable)); + } else { + expectedParams.put("ignore_unavailable", Boolean.FALSE.toString()); + } + + if (randomBoolean()) { + boolean verbose = randomBoolean(); + getSnapshotsRequest.verbose(verbose); + expectedParams.put("verbose", Boolean.toString(verbose)); + } else { + expectedParams.put("verbose", Boolean.TRUE.toString()); + } + + Request request = RequestConverters.getSnapshots(getSnapshotsRequest); + assertThat(endpoint, equalTo(request.getEndpoint())); + assertThat(HttpGet.METHOD_NAME, equalTo(request.getMethod())); + assertThat(expectedParams, equalTo(request.getParameters())); + assertNull(request.getEntity()); + } + + public void testGetAllSnapshots() { + Map expectedParams = new HashMap<>(); + String repository = randomIndicesNames(1, 1)[0]; + + String endpoint = String.format(Locale.ROOT, "/_snapshot/%s/_all", repository); + + GetSnapshotsRequest getSnapshotsRequest = new GetSnapshotsRequest(repository); + setRandomMasterTimeout(getSnapshotsRequest, expectedParams); + + boolean ignoreUnavailable = randomBoolean(); + getSnapshotsRequest.ignoreUnavailable(ignoreUnavailable); + expectedParams.put("ignore_unavailable", Boolean.toString(ignoreUnavailable)); + + boolean verbose = randomBoolean(); + getSnapshotsRequest.verbose(verbose); + expectedParams.put("verbose", Boolean.toString(verbose)); + + Request request = RequestConverters.getSnapshots(getSnapshotsRequest); + assertThat(endpoint, equalTo(request.getEndpoint())); + assertThat(HttpGet.METHOD_NAME, equalTo(request.getMethod())); + assertThat(expectedParams, equalTo(request.getParameters())); + assertNull(request.getEntity()); + } + public void testDeleteSnapshot() { Map expectedParams = new HashMap<>(); String repository = randomIndicesNames(1, 1)[0]; diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java index aacb2f5025ee4..7ec2ee80f04ac 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SnapshotIT.java @@ -32,12 +32,16 @@ import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest; import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotResponse; +import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; +import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.repositories.fs.FsRepository; import org.elasticsearch.rest.RestStatus; import java.io.IOException; +import java.util.stream.Collectors; +import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; public class SnapshotIT extends ESRestHighLevelClientTestCase { @@ -135,6 +139,40 @@ public void testCreateSnapshot() throws IOException { assertEquals(waitForCompletion ? RestStatus.OK : RestStatus.ACCEPTED, response.status()); } + public void testGetSnapshots() throws IOException { + String repository = "test_repository"; + String snapshot1 = "test_snapshot1"; + String snapshot2 = "test_snapshot2"; + + PutRepositoryResponse putRepositoryResponse = createTestRepository(repository, FsRepository.TYPE, "{\"location\": \".\"}"); + assertTrue(putRepositoryResponse.isAcknowledged()); + + CreateSnapshotRequest createSnapshotRequest1 = new CreateSnapshotRequest(repository, snapshot1); + createSnapshotRequest1.waitForCompletion(true); + CreateSnapshotResponse putSnapshotResponse1 = createTestSnapshot(createSnapshotRequest1); + CreateSnapshotRequest createSnapshotRequest2 = new CreateSnapshotRequest(repository, snapshot2); + createSnapshotRequest2.waitForCompletion(true); + CreateSnapshotResponse putSnapshotResponse2 = createTestSnapshot(createSnapshotRequest2); + // check that the request went ok without parsing JSON here. When using the high level client, check acknowledgement instead. + assertEquals(RestStatus.OK, putSnapshotResponse1.status()); + assertEquals(RestStatus.OK, putSnapshotResponse2.status()); + + GetSnapshotsRequest request; + if (randomBoolean()) { + request = new GetSnapshotsRequest(repository); + } else if (randomBoolean()) { + request = new GetSnapshotsRequest(repository, new String[] {"_all"}); + + } else { + request = new GetSnapshotsRequest(repository, new String[] {snapshot1, snapshot2}); + } + GetSnapshotsResponse response = execute(request, highLevelClient().snapshot()::get, highLevelClient().snapshot()::getAsync); + + assertEquals(2, response.getSnapshots().size()); + assertThat(response.getSnapshots().stream().map((s) -> s.snapshotId().getName()).collect(Collectors.toList()), + contains("test_snapshot1", "test_snapshot2")); + } + public void testDeleteSnapshot() throws IOException { String repository = "test_repository"; String snapshot = "test_snapshot"; diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java index 9c0e31bdcfb70..48d01963e236d 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SnapshotClientDocumentationIT.java @@ -31,6 +31,8 @@ import org.elasticsearch.action.admin.cluster.repositories.verify.VerifyRepositoryResponse; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest; import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse; +import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest; +import org.elasticsearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse; import org.elasticsearch.action.admin.indices.create.CreateIndexRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequest; @@ -46,6 +48,10 @@ import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.repositories.fs.FsRepository; import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.snapshots.SnapshotId; +import org.elasticsearch.snapshots.SnapshotInfo; +import org.elasticsearch.snapshots.SnapshotShardFailure; +import org.elasticsearch.snapshots.SnapshotState; import java.io.IOException; import java.util.HashMap; @@ -456,6 +462,87 @@ public void onFailure(Exception exception) { } } + public void testSnapshotGetSnapshots() throws IOException { + RestHighLevelClient client = highLevelClient(); + + createTestRepositories(); + createTestSnapshots(); + + // tag::get-snapshots-request + GetSnapshotsRequest request = new GetSnapshotsRequest(); + // end::get-snapshots-request + + // tag::get-snapshots-request-repositoryName + request.repository(repositoryName); // <1> + // end::get-snapshots-request-repositoryName + + // tag::get-snapshots-request-snapshots + String[] snapshots = { snapshotName }; + request.snapshots(snapshots); // <1> + // end::get-snapshots-request-snapshots + + // tag::get-snapshots-request-masterTimeout + request.masterNodeTimeout(TimeValue.timeValueMinutes(1)); // <1> + request.masterNodeTimeout("1m"); // <2> + // end::get-snapshots-request-masterTimeout + + // tag::get-snapshots-request-verbose + request.verbose(true); // <1> + // end::get-snapshots-request-verbose + + // tag::get-snapshots-request-ignore-unavailable + request.ignoreUnavailable(false); // <1> + // end::get-snapshots-request-ignore-unavailable + + // tag::get-snapshots-execute + GetSnapshotsResponse response = client.snapshot().get(request, RequestOptions.DEFAULT); + // end::get-snapshots-execute + + // tag::get-snapshots-response + List snapshotsInfos = response.getSnapshots(); + SnapshotInfo snapshotInfo = snapshotsInfos.get(0); + RestStatus restStatus = snapshotInfo.status(); // <1> + SnapshotId snapshotId = snapshotInfo.snapshotId(); // <2> + SnapshotState snapshotState = snapshotInfo.state(); // <3> + List snapshotShardFailures = snapshotInfo.shardFailures(); // <4> + long startTime = snapshotInfo.startTime(); // <5> + long endTime = snapshotInfo.endTime(); // <6> + // end::get-snapshots-response + assertEquals(1, snapshotsInfos.size()); + } + + public void testSnapshotGetSnapshotsAsync() throws InterruptedException { + RestHighLevelClient client = highLevelClient(); + { + GetSnapshotsRequest request = new GetSnapshotsRequest(repositoryName); + + // tag::get-snapshots-execute-listener + ActionListener listener = + new ActionListener() { + @Override + public void onResponse(GetSnapshotsResponse getSnapshotsResponse) { + // <1> + } + + @Override + public void onFailure(Exception e) { + // <2> + } + }; + // end::get-snapshots-execute-listener + + // Replace the empty listener by a blocking listener in test + final CountDownLatch latch = new CountDownLatch(1); + listener = new LatchedActionListener<>(listener, latch); + + // tag::get-snapshots-execute-async + client.snapshot().getAsync(request, RequestOptions.DEFAULT, listener); // <1> + // end::get-snapshots-execute-async + + assertTrue(latch.await(30L, TimeUnit.SECONDS)); + } + } + public void testSnapshotDeleteSnapshot() throws IOException { RestHighLevelClient client = highLevelClient(); diff --git a/docs/java-rest/high-level/snapshot/get_snapshots.asciidoc b/docs/java-rest/high-level/snapshot/get_snapshots.asciidoc new file mode 100644 index 0000000000000..1041f61d8422b --- /dev/null +++ b/docs/java-rest/high-level/snapshot/get_snapshots.asciidoc @@ -0,0 +1,108 @@ +[[java-rest-high-snapshot-get-snapshots]] +=== Get Snapshots API + +Use the Get Snapshot API to get snapshots. + +[[java-rest-high-snapshot-get-snapshots-request]] +==== Get Snapshots Request + +A `GetSnapshotsRequest`: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[get-snapshots-request] +-------------------------------------------------- + +==== Required Arguments +The following arguments are mandatory: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[get-snapshots-request-repositoryName] +-------------------------------------------------- +<1> The name of the repository. + +==== Optional Arguments +The following arguments are optional: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[get-snapshots-request-snapshots] +-------------------------------------------------- +<1> An array of snapshots to get. Otherwise it will return all snapshots for a repository. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[get-snapshots-request-masterTimeout] +-------------------------------------------------- +<1> Timeout to connect to the master node as a `TimeValue`. +<2> Timeout to connect to the master node as a `String`. + + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[get-snapshots-request-verbose] +-------------------------------------------------- +<1> `Boolean` indicating if the response should be verbose. + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[get-snapshots-request-ignore-unavailable] +-------------------------------------------------- +<1> `Boolean` indicating if unavailable snapshots should be ignored. Otherwise the request will +fail if any of the snapshots are unavailable. + +[[java-rest-high-snapshot-get-snapshots-sync]] +==== Synchronous Execution + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[get-snapshots-execute] +-------------------------------------------------- + +[[java-rest-high-snapshot-get-snapshots-async]] +==== Asynchronous Execution + +The asynchronous execution of a get snapshots request requires both the +`GetSnapshotsRequest` instance and an `ActionListener` instance to be +passed as arguments to the asynchronous method: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[get-snapshots-execute-async] +-------------------------------------------------- +<1> The `GetSnapshotsRequest` to execute and the `ActionListener` to use when +the execution completes. + +The asynchronous method does not block and returns immediately. Once it is +completed the `ActionListener` is called back with the `onResponse` method +if the execution is successful or the `onFailure` method if the execution +failed. + +A typical listener for `GetSnapshotsResponse` looks like: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[get-snapshots-execute-listener] +-------------------------------------------------- +<1> Called when the execution is successfully completed. The response is +provided as an argument. +<2> Called in case of a failure. The raised exception is provided as an +argument. + +[[java-rest-high-snapshot-get-snapshots-response]] +==== Get Snapshots Response + +The returned `GetSnapshotsResponse` allows the retrieval of information about the requested +snapshots: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests}/SnapshotClientDocumentationIT.java[get-snapshots-response] +-------------------------------------------------- +<1> The REST status of a snapshot +<2> The snapshot id +<3> The current state of the snapshot +<4> Information about failures that occurred during the shard snapshot process. +<5> The snapshot start time +<6> The snapshot end time \ No newline at end of file diff --git a/docs/java-rest/high-level/supported-apis.asciidoc b/docs/java-rest/high-level/supported-apis.asciidoc index 447c984adf380..e69f53eb4ba41 100644 --- a/docs/java-rest/high-level/supported-apis.asciidoc +++ b/docs/java-rest/high-level/supported-apis.asciidoc @@ -153,6 +153,7 @@ The Java High Level REST Client supports the following Snapshot APIs: * <> * <> * <> +* <> * <> include::snapshot/get_repository.asciidoc[] @@ -160,6 +161,7 @@ include::snapshot/create_repository.asciidoc[] include::snapshot/delete_repository.asciidoc[] include::snapshot/verify_repository.asciidoc[] include::snapshot/create_snapshot.asciidoc[] +include::snapshot/get_snapshots.asciidoc[] include::snapshot/delete_snapshot.asciidoc[] == Tasks APIs diff --git a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java index 0d1e5eda7f2d2..6f757cb60ca86 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java +++ b/server/src/main/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponse.java @@ -20,23 +20,37 @@ package org.elasticsearch.action.admin.cluster.snapshots.get; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.snapshots.SnapshotInfo; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Objects; /** * Get snapshots response */ public class GetSnapshotsResponse extends ActionResponse implements ToXContentObject { + @SuppressWarnings("unchecked") + private static final ConstructingObjectParser GET_SNAPSHOT_PARSER = + new ConstructingObjectParser<>(GetSnapshotsResponse.class.getName(), true, + (args) -> new GetSnapshotsResponse((List) args[0])); + + static { + GET_SNAPSHOT_PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), + (p, c) -> SnapshotInfo.SNAPSHOT_INFO_PARSER.apply(p, c).build(), new ParseField("snapshots")); + } + private List snapshots = Collections.emptyList(); GetSnapshotsResponse() { @@ -87,4 +101,20 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par return builder; } + public static GetSnapshotsResponse fromXContent(XContentParser parser) throws IOException { + return GET_SNAPSHOT_PARSER.parse(parser, null); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + GetSnapshotsResponse that = (GetSnapshotsResponse) o; + return Objects.equals(snapshots, that.snapshots); + } + + @Override + public int hashCode() { + return Objects.hash(snapshots); + } } diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java index ddd7385056d55..a1f56a1e47376 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotInfo.java @@ -31,15 +31,12 @@ import org.elasticsearch.common.joda.Joda; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ObjectParser; -import org.elasticsearch.common.xcontent.ObjectParser.ValueType; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.rest.RestStatus; import java.io.IOException; -import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -84,7 +81,7 @@ public final class SnapshotInfo implements Comparable, ToXContent, private static final Comparator COMPARATOR = Comparator.comparing(SnapshotInfo::startTime).thenComparing(SnapshotInfo::snapshotId); - private static final class SnapshotInfoBuilder { + public static final class SnapshotInfoBuilder { private String snapshotName = null; private String snapshotUUID = null; private String state = null; @@ -137,23 +134,8 @@ private void setVersion(int version) { this.version = version; } - private void setShardFailures(XContentParser parser) { - if (shardFailures == null) { - shardFailures = new ArrayList<>(); - } - - try { - if (parser.currentToken() == Token.START_ARRAY) { - parser.nextToken(); - } - - while (parser.currentToken() != Token.END_ARRAY) { - shardFailures.add(SnapshotShardFailure.fromXContent(parser)); - parser.nextToken(); - } - } catch (IOException exception) { - throw new UncheckedIOException(exception); - } + private void setShardFailures(List shardFailures) { + this.shardFailures = shardFailures; } private void ignoreVersion(String version) { @@ -172,7 +154,7 @@ private void ignoreDurationInMillis(long durationInMillis) { // ignore extra field } - private SnapshotInfo build() { + public SnapshotInfo build() { SnapshotId snapshotId = new SnapshotId(snapshotName, snapshotUUID); if (indices == null) { @@ -219,11 +201,11 @@ private void ignoreFailedShards(int failedShards) { } } - private static final ObjectParser SNAPSHOT_INFO_PARSER = - new ObjectParser<>(SnapshotInfoBuilder.class.getName(), SnapshotInfoBuilder::new); + public static final ObjectParser SNAPSHOT_INFO_PARSER = + new ObjectParser<>(SnapshotInfoBuilder.class.getName(), true, SnapshotInfoBuilder::new); private static final ObjectParser SHARD_STATS_PARSER = - new ObjectParser<>(ShardStatsBuilder.class.getName(), ShardStatsBuilder::new); + new ObjectParser<>(ShardStatsBuilder.class.getName(), true, ShardStatsBuilder::new); static { SNAPSHOT_INFO_PARSER.declareString(SnapshotInfoBuilder::setSnapshotName, new ParseField(SNAPSHOT)); @@ -236,8 +218,8 @@ private void ignoreFailedShards(int failedShards) { SNAPSHOT_INFO_PARSER.declareObject(SnapshotInfoBuilder::setShardStatsBuilder, SHARD_STATS_PARSER, new ParseField(SHARDS)); SNAPSHOT_INFO_PARSER.declareBoolean(SnapshotInfoBuilder::setIncludeGlobalState, new ParseField(INCLUDE_GLOBAL_STATE)); SNAPSHOT_INFO_PARSER.declareInt(SnapshotInfoBuilder::setVersion, new ParseField(VERSION_ID)); - SNAPSHOT_INFO_PARSER.declareField( - SnapshotInfoBuilder::setShardFailures, parser -> parser, new ParseField(FAILURES), ValueType.OBJECT_ARRAY_OR_STRING); + SNAPSHOT_INFO_PARSER.declareObjectArray(SnapshotInfoBuilder::setShardFailures, SnapshotShardFailure.SNAPSHOT_SHARD_FAILURE_PARSER, + new ParseField(FAILURES)); SNAPSHOT_INFO_PARSER.declareString(SnapshotInfoBuilder::ignoreVersion, new ParseField(VERSION)); SNAPSHOT_INFO_PARSER.declareString(SnapshotInfoBuilder::ignoreStartTime, new ParseField(START_TIME)); SNAPSHOT_INFO_PARSER.declareString(SnapshotInfoBuilder::ignoreEndTime, new ParseField(END_TIME)); @@ -521,7 +503,7 @@ public RestStatus status() { public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { // write snapshot info to repository snapshot blob format if (CONTEXT_MODE_SNAPSHOT.equals(params.param(CONTEXT_MODE_PARAM))) { - return toXContentSnapshot(builder, params); + return toXContentInternal(builder, params); } final boolean verbose = params.paramAsBoolean("verbose", GetSnapshotsRequest.DEFAULT_VERBOSE_MODE); @@ -576,7 +558,7 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa return builder; } - private XContentBuilder toXContentSnapshot(final XContentBuilder builder, final ToXContent.Params params) throws IOException { + private XContentBuilder toXContentInternal(final XContentBuilder builder, final ToXContent.Params params) throws IOException { builder.startObject(SNAPSHOT); builder.field(NAME, snapshotId.getName()); builder.field(UUID, snapshotId.getUUID()); @@ -609,22 +591,12 @@ private XContentBuilder toXContentSnapshot(final XContentBuilder builder, final return builder; } + /** + * This method creates a SnapshotInfo from external x-content. It does not + * handle x-content written with the internal version. + */ public static SnapshotInfo fromXContent(final XContentParser parser) throws IOException { - parser.nextToken(); // // move to '{' - - if (parser.currentToken() != Token.START_OBJECT) { - throw new IllegalArgumentException("unexpected token [" + parser.currentToken() + "], expected ['{']"); - } - - SnapshotInfo snapshotInfo = SNAPSHOT_INFO_PARSER.apply(parser, null).build(); - - if (parser.currentToken() != Token.END_OBJECT) { - throw new IllegalArgumentException("unexpected token [" + parser.currentToken() + "], expected ['}']"); - } - - parser.nextToken(); // move past '}' - - return snapshotInfo; + return SNAPSHOT_INFO_PARSER.parse(parser, null).build(); } /** diff --git a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardFailure.java b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardFailure.java index 30057a6220dcb..1578cb0a289ee 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardFailure.java +++ b/server/src/main/java/org/elasticsearch/snapshots/SnapshotShardFailure.java @@ -23,8 +23,10 @@ import org.elasticsearch.action.ShardOperationFailedException; import org.elasticsearch.cluster.metadata.IndexMetaData; import org.elasticsearch.common.Nullable; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -60,11 +62,23 @@ private SnapshotShardFailure() { * @param reason failure reason */ public SnapshotShardFailure(@Nullable String nodeId, ShardId shardId, String reason) { + this(nodeId, shardId, reason, RestStatus.INTERNAL_SERVER_ERROR); + } + + /** + * Constructs new snapshot shard failure object + * + * @param nodeId node where failure occurred + * @param shardId shard id + * @param reason failure reason + * @param status rest status + */ + private SnapshotShardFailure(@Nullable String nodeId, ShardId shardId, String reason, RestStatus status) { + assert reason != null; this.nodeId = nodeId; this.shardId = shardId; this.reason = reason; - assert reason != null; - status = RestStatus.INTERNAL_SERVER_ERROR; + this.status = status; } /** @@ -98,7 +112,7 @@ public String reason() { } /** - * Returns REST status corresponding to this failure + * Returns {@link RestStatus} corresponding to this failure * * @return REST status */ @@ -173,63 +187,65 @@ public static void toXContent(SnapshotShardFailure snapshotShardFailure, XConten builder.endObject(); } - /** - * Deserializes snapshot failure information from JSON - * - * @param parser JSON parser - * @return snapshot failure information - */ - public static SnapshotShardFailure fromXContent(XContentParser parser) throws IOException { - SnapshotShardFailure snapshotShardFailure = new SnapshotShardFailure(); - - XContentParser.Token token = parser.currentToken(); - String index = null; - String index_uuid = IndexMetaData.INDEX_UUID_NA_VALUE; - int shardId = -1; - if (token == XContentParser.Token.START_OBJECT) { - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - String currentFieldName = parser.currentName(); - token = parser.nextToken(); - if (token.isValue()) { - if ("index".equals(currentFieldName)) { - index = parser.text(); - } else if ("index_uuid".equals(currentFieldName)) { - index_uuid = parser.text(); - } else if ("node_id".equals(currentFieldName)) { - snapshotShardFailure.nodeId = parser.text(); - } else if ("reason".equals(currentFieldName)) { - // Workaround for https://github.com/elastic/elasticsearch/issues/25878 - // Some old snapshot might still have null in shard failure reasons - snapshotShardFailure.reason = parser.textOrNull(); - } else if ("shard_id".equals(currentFieldName)) { - shardId = parser.intValue(); - } else if ("status".equals(currentFieldName)) { - snapshotShardFailure.status = RestStatus.valueOf(parser.text()); - } else { - throw new ElasticsearchParseException("unknown parameter [{}]", currentFieldName); - } - } - } else { - throw new ElasticsearchParseException("unexpected token [{}]", token); - } - } - } else { - throw new ElasticsearchParseException("unexpected token [{}]", token); - } + static final ConstructingObjectParser SNAPSHOT_SHARD_FAILURE_PARSER = + new ConstructingObjectParser<>("shard_failure", true, SnapshotShardFailure::constructSnapshotShardFailure); + + static { + SNAPSHOT_SHARD_FAILURE_PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField("index")); + SNAPSHOT_SHARD_FAILURE_PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), new ParseField("index_uuid")); + SNAPSHOT_SHARD_FAILURE_PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), new ParseField("node_id")); + // Workaround for https://github.com/elastic/elasticsearch/issues/25878 + // Some old snapshot might still have null in shard failure reasons + SNAPSHOT_SHARD_FAILURE_PARSER.declareStringOrNull(ConstructingObjectParser.optionalConstructorArg(), new ParseField("reason")); + SNAPSHOT_SHARD_FAILURE_PARSER.declareInt(ConstructingObjectParser.constructorArg(), new ParseField("shard_id")); + SNAPSHOT_SHARD_FAILURE_PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), new ParseField("status")); + } + + private static SnapshotShardFailure constructSnapshotShardFailure(Object[] args) { + String index = (String) args[0]; + String indexUuid = (String) args[1]; + String nodeId = (String) args[2]; + String reason = (String) args[3]; + Integer intShardId = (Integer) args[4]; + String status = (String) args[5]; + if (index == null) { throw new ElasticsearchParseException("index name was not set"); } - if (shardId == -1) { + if (intShardId == null) { throw new ElasticsearchParseException("index shard was not set"); } - snapshotShardFailure.shardId = new ShardId(index, index_uuid, shardId); + + ShardId shardId = new ShardId(index, indexUuid != null ? indexUuid : IndexMetaData.INDEX_UUID_NA_VALUE, intShardId); + // Workaround for https://github.com/elastic/elasticsearch/issues/25878 // Some old snapshot might still have null in shard failure reasons - if (snapshotShardFailure.reason == null) { - snapshotShardFailure.reason = ""; + String nonNullReason; + if (reason != null) { + nonNullReason = reason; + } else { + nonNullReason = ""; + } + + + RestStatus restStatus; + if (status != null) { + restStatus = RestStatus.valueOf(status); + } else { + restStatus = RestStatus.INTERNAL_SERVER_ERROR; } - return snapshotShardFailure; + + return new SnapshotShardFailure(nodeId, shardId, nonNullReason, restStatus); + } + + /** + * Deserializes snapshot failure information from JSON + * + * @param parser JSON parser + * @return snapshot failure information + */ + public static SnapshotShardFailure fromXContent(XContentParser parser) throws IOException { + return SNAPSHOT_SHARD_FAILURE_PARSER.parse(parser, null); } @Override diff --git a/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java new file mode 100644 index 0000000000000..c5bd7d9f38ac1 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/action/admin/cluster/snapshots/get/GetSnapshotsResponseTests.java @@ -0,0 +1,62 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.action.admin.cluster.snapshots.get; + +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.snapshots.SnapshotId; +import org.elasticsearch.snapshots.SnapshotInfo; +import org.elasticsearch.snapshots.SnapshotShardFailure; +import org.elasticsearch.test.AbstractStreamableXContentTestCase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +public class GetSnapshotsResponseTests extends AbstractStreamableXContentTestCase { + + @Override + protected GetSnapshotsResponse doParseInstance(XContentParser parser) throws IOException { + return GetSnapshotsResponse.fromXContent(parser); + } + + @Override + protected GetSnapshotsResponse createBlankInstance() { + return new GetSnapshotsResponse(); + } + + @Override + protected GetSnapshotsResponse createTestInstance() { + ArrayList snapshots = new ArrayList<>(); + for (int i = 0; i < randomIntBetween(5, 10); ++i) { + SnapshotId snapshotId = new SnapshotId("snapshot " + i, UUIDs.base64UUID()); + String reason = randomBoolean() ? null : "reason"; + ShardId shardId = new ShardId("index", UUIDs.base64UUID(), 2); + List shardFailures = Collections.singletonList(new SnapshotShardFailure("node-id", shardId, "reason")); + snapshots.add(new SnapshotInfo(snapshotId, Arrays.asList("indice1", "indice2"), System.currentTimeMillis(), reason, + System.currentTimeMillis(), randomIntBetween(2, 3), shardFailures, randomBoolean())); + + } + return new GetSnapshotsResponse(snapshots); + } +} From ced20affb100bdc186de4c350dbcadfdfe48a6b1 Mon Sep 17 00:00:00 2001 From: olcbean Date: Thu, 12 Jul 2018 19:10:51 +0200 Subject: [PATCH 46/50] XContentTests : Insert random fields at random positions (#30867) Currently AbstractXContentTestCase#testFromXContent appends random fields, but in a fixed position. This PR shuffles all fields after the random fields have been appended, hence the random fields are actually added to random positions. --- .../test/AbstractXContentTestCase.java | 38 +++++++---- .../test/AbstractXContentTestCaseTests.java | 65 +++++++++++++++++++ 2 files changed, 90 insertions(+), 13 deletions(-) create mode 100644 test/framework/src/test/java/org/elasticsearch/test/AbstractXContentTestCaseTests.java diff --git a/test/framework/src/main/java/org/elasticsearch/test/AbstractXContentTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/AbstractXContentTestCase.java index fd5700c68a981..c740a65d28a6e 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/AbstractXContentTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/AbstractXContentTestCase.java @@ -54,21 +54,16 @@ public static void testFromXContent(int numberOfTestRuns, for (int runs = 0; runs < numberOfTestRuns; runs++) { T testInstance = instanceSupplier.get(); XContentType xContentType = randomFrom(XContentType.values()); - BytesReference shuffled = toShuffledXContent(testInstance, xContentType, toXContentParams,false, - createParserFunction, shuffleFieldsExceptions); - BytesReference withRandomFields; - if (supportsUnknownFields) { - // we add a few random fields to check that parser is lenient on new fields - withRandomFields = XContentTestUtils.insertRandomFields(xContentType, shuffled, randomFieldsExcludeFilter, random()); - } else { - withRandomFields = shuffled; - } - XContentParser parser = createParserFunction.apply(XContentFactory.xContent(xContentType), withRandomFields); + BytesReference shuffledContent = insertRandomFieldsAndShuffle(testInstance, xContentType, supportsUnknownFields, + shuffleFieldsExceptions, randomFieldsExcludeFilter, createParserFunction, toXContentParams); + XContentParser parser = createParserFunction.apply(XContentFactory.xContent(xContentType), shuffledContent); T parsed = parseFunction.apply(parser); assertEqualsConsumer.accept(testInstance, parsed); if (assertToXContentEquivalence) { - assertToXContentEquivalent(shuffled, XContentHelper.toXContent(parsed, xContentType, toXContentParams, false), - xContentType); + assertToXContentEquivalent( + XContentHelper.toXContent(testInstance, xContentType, toXContentParams, false), + XContentHelper.toXContent(parsed, xContentType, toXContentParams, false), + xContentType); } } } @@ -132,9 +127,26 @@ protected String[] getShuffleFieldsExceptions() { } /** - * Params that have to be provided when calling calling {@link ToXContent#toXContent(XContentBuilder, ToXContent.Params)} + * Params that have to be provided when calling {@link ToXContent#toXContent(XContentBuilder, ToXContent.Params)} */ protected ToXContent.Params getToXContentParams() { return ToXContent.EMPTY_PARAMS; } + + static BytesReference insertRandomFieldsAndShuffle(ToXContent testInstance, XContentType xContentType, + boolean supportsUnknownFields, String[] shuffleFieldsExceptions, Predicate randomFieldsExcludeFilter, + CheckedBiFunction createParserFunction, + ToXContent.Params toXContentParams) throws IOException { + BytesReference xContent = XContentHelper.toXContent(testInstance, xContentType, toXContentParams, false); + BytesReference withRandomFields; + if (supportsUnknownFields) { + // add a few random fields to check that the parser is lenient on new fields + withRandomFields = XContentTestUtils.insertRandomFields(xContentType, xContent, randomFieldsExcludeFilter, random()); + } else { + withRandomFields = xContent; + } + XContentParser parserWithRandonFields = createParserFunction.apply(XContentFactory.xContent(xContentType), withRandomFields); + return BytesReference.bytes(ESTestCase.shuffleXContent(parserWithRandonFields, false, shuffleFieldsExceptions)); + } + } diff --git a/test/framework/src/test/java/org/elasticsearch/test/AbstractXContentTestCaseTests.java b/test/framework/src/test/java/org/elasticsearch/test/AbstractXContentTestCaseTests.java new file mode 100644 index 0000000000000..165df584c030c --- /dev/null +++ b/test/framework/src/test/java/org/elasticsearch/test/AbstractXContentTestCaseTests.java @@ -0,0 +1,65 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.test; + +import com.carrotsearch.randomizedtesting.RandomizedContext; + +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; + +import java.io.IOException; +import java.util.Map; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; + +public class AbstractXContentTestCaseTests extends ESTestCase { + + public void testInsertRandomFieldsAndShuffle() throws Exception { + TestInstance t = new TestInstance(); + BytesReference insertRandomFieldsAndShuffle = RandomizedContext.current().runWithPrivateRandomness(1, + () -> AbstractXContentTestCase.insertRandomFieldsAndShuffle(t, XContentType.JSON, true, new String[] {}, null, + this::createParser, ToXContent.EMPTY_PARAMS)); + try (XContentParser parser = createParser(XContentType.JSON.xContent(), insertRandomFieldsAndShuffle)) { + Map mapOrdered = parser.mapOrdered(); + assertThat(mapOrdered.size(), equalTo(2)); + assertThat(mapOrdered.keySet().iterator().next(), not(equalTo("field"))); + } + } + + private class TestInstance implements ToXContentObject { + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + { + builder.field("field", 1); + } + builder.endObject(); + return builder; + } + + } + +} \ No newline at end of file From 686bf28537a7e7e0c10e468a5de657d114148443 Mon Sep 17 00:00:00 2001 From: Andy Bristol Date: Thu, 12 Jul 2018 10:55:55 -0700 Subject: [PATCH 47/50] [test] disable java packaging tests for suse For #30295 --- .../elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy index 90daaccd6a749..9aa762e53cb7e 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy @@ -569,7 +569,11 @@ class VagrantTestPlugin implements Plugin { project.gradle.removeListener(javaPackagingReproListener) } if (project.extensions.esvagrant.boxes.contains(box)) { - packagingTest.dependsOn(javaPackagingTest) + // these tests are temporarily disabled for suse boxes while we debug an issue + // https://github.com/elastic/elasticsearch/issues/30295 + if (box.equals("opensuse-42") == false && box.equals("sles-12") == false) { + packagingTest.dependsOn(javaPackagingTest) + } } /* From e052091d46b6655456e8bae4ccc32eb0d011823d Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 12 Jul 2018 16:22:42 -0400 Subject: [PATCH 48/50] Docs: Explain closing the high level client It looks like we weren't clear on when and why you should close the high level client and folks were closing it after every request which is not efficient. This explains why you should close the client and when so this shouldn't be as common. Closes #32001 --- docs/java-rest/high-level/getting-started.asciidoc | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/java-rest/high-level/getting-started.asciidoc b/docs/java-rest/high-level/getting-started.asciidoc index 3e9b9fa7ea08f..af03a6a86bbd5 100644 --- a/docs/java-rest/high-level/getting-started.asciidoc +++ b/docs/java-rest/high-level/getting-started.asciidoc @@ -130,12 +130,11 @@ include-tagged::{doc-tests}/MiscellaneousDocumentationIT.java[rest-high-level-cl -------------------------------------------------- The high-level client will internally create the low-level client used to -perform requests based on the provided builder, and manage its lifecycle. - -The high-level client instance needs to be closed when no longer needed so that -all the resources used by it get properly released, as well as the underlying -http client instance and its threads. This can be done through the `close` -method, which will close the internal `RestClient` instance. +perform requests based on the provided builder. That low-level client +maintains a pool of connections and starts some threads so you should +close the high-level client when you are well and truly done with +it and it will in turn close the internal low-level client to free those +resources. This can be done through the `close`: ["source","java",subs="attributes,callouts,macros"] -------------------------------------------------- From fb6cce0fb4a64b183393ad62cf4020fe6e808e6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 12 Jul 2018 22:41:35 +0200 Subject: [PATCH 49/50] [TEST] Mute SlackMessageTests.testTemplateRender --- .../watcher/notification/slack/message/SlackMessageTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/slack/message/SlackMessageTests.java b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/slack/message/SlackMessageTests.java index 83e2e9978391a..14d732064e5e0 100644 --- a/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/slack/message/SlackMessageTests.java +++ b/x-pack/plugin/watcher/src/test/java/org/elasticsearch/xpack/watcher/notification/slack/message/SlackMessageTests.java @@ -461,6 +461,7 @@ public void testTemplateParseSelfGenerated() throws Exception { assertThat(parsed, equalTo(template)); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/31948") public void testTemplateRender() throws Exception { Settings settings = SlackMessageDefaultsTests.randomSettings(); SlackMessageDefaults defaults = new SlackMessageDefaults(settings); From 71380bd9e615970f11d5f85575fd28dde09b0989 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Thu, 12 Jul 2018 08:56:06 -0700 Subject: [PATCH 50/50] Force execution of fetch tasks (#31974) Forces fetch tasks to queue even in the event that the queue is already full. The reasoning is that fetch tasks may only be follow-up to query tasks, so the number of additional fetch tasks that may enter the threadpool is expected to be reasonable. Closes #29442 --- .../org/elasticsearch/action/search/SearchTransportService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java index 696b1e9a9aa8c..dc47117822476 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchTransportService.java @@ -432,7 +432,7 @@ public void messageReceived(ShardFetchRequest request, TransportChannel channel, }); TransportActionProxy.registerProxyAction(transportService, FETCH_ID_SCROLL_ACTION_NAME, FetchSearchResult::new); - transportService.registerRequestHandler(FETCH_ID_ACTION_NAME, ShardFetchSearchRequest::new, ThreadPool.Names.SEARCH, + transportService.registerRequestHandler(FETCH_ID_ACTION_NAME, ShardFetchSearchRequest::new, ThreadPool.Names.SEARCH, true, true, new TaskAwareTransportRequestHandler() { @Override public void messageReceived(ShardFetchSearchRequest request, TransportChannel channel, Task task) throws Exception {