From ca6808e55de5b9b5d0daa074c1512366fcb57fde Mon Sep 17 00:00:00 2001 From: Costin Leau Date: Tue, 23 Oct 2018 17:07:51 +0300 Subject: [PATCH] SQL: Support pattern against compatible indices (#34718) Extend querying support on multiple indices from being strictly identical to being just compatible. Use FieldCapabilities API (extended through #33803) for mapping merging. Close #31837 #31611 --- docs/reference/sql/security.asciidoc | 2 +- .../xpack/sql/type/DataType.java | 10 +- .../sql/analysis/index/IndexResolver.java | 196 ++++++++++++++---- .../sql/plan/logical/command/ShowColumns.java | 11 +- .../querydsl/container/QueryContainer.java | 6 +- .../xpack/sql/session/SqlSession.java | 2 +- .../elasticsearch/xpack/sql/type/EsField.java | 36 ++-- .../analysis/index/IndexResolverTests.java | 189 ++++++++++++++--- .../xpack/sql/type/TypesTests.java | 14 +- .../resources/mapping-basic-incompatible.json | 22 ++ .../resources/mapping-basic-nodocvalues.json | 23 ++ x-pack/qa/sql/security/roles.yml | 10 +- .../xpack/qa/sql/security/CliSecurityIT.java | 23 +- .../xpack/qa/sql/security/JdbcSecurityIT.java | 20 +- .../qa/sql/security/RestSqlSecurityIT.java | 14 +- .../qa/sql/security/SqlSecurityTestCase.java | 103 +++++---- .../xpack/qa/sql/cli/EmbeddedCli.java | 10 +- .../xpack/qa/sql/cli/ErrorsTestCase.java | 6 +- .../xpack/qa/sql/jdbc/DataLoader.java | 30 ++- .../xpack/qa/sql/jdbc/ErrorsTestCase.java | 7 +- .../xpack/qa/sql/rest/RestSqlTestCase.java | 4 +- .../qa/sql/src/main/resources/alias.csv-spec | 76 +++---- .../sql/src/main/resources/command.csv-spec | 153 +++++++------- .../qa/sql/src/main/resources/docs.csv-spec | 72 +++---- .../qa/sql/src/main/resources/nested.csv-spec | 36 ++-- 25 files changed, 727 insertions(+), 348 deletions(-) create mode 100644 x-pack/plugin/sql/src/test/resources/mapping-basic-incompatible.json create mode 100644 x-pack/plugin/sql/src/test/resources/mapping-basic-nodocvalues.json diff --git a/docs/reference/sql/security.asciidoc b/docs/reference/sql/security.asciidoc index 64f554f023195..a317355866b8b 100644 --- a/docs/reference/sql/security.asciidoc +++ b/docs/reference/sql/security.asciidoc @@ -34,6 +34,6 @@ indices: ["source","yaml",subs="attributes,callouts,macros"] -------------------------------------------------- -include-tagged::{sql-tests}/security/roles.yml[cli_jdbc] +include-tagged::{sql-tests}/security/roles.yml[cli_drivers] -------------------------------------------------- diff --git a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/type/DataType.java b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/type/DataType.java index 1c08c6e1c9fa1..1c9cf6ac9257f 100644 --- a/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/type/DataType.java +++ b/x-pack/plugin/sql/sql-proto/src/main/java/org/elasticsearch/xpack/sql/type/DataType.java @@ -36,8 +36,8 @@ public enum DataType { SCALED_FLOAT(JDBCType.FLOAT, Double.class, Double.BYTES, 19, 25, false, true, true), KEYWORD( JDBCType.VARCHAR, String.class, Integer.MAX_VALUE, 256, 0), TEXT( JDBCType.VARCHAR, String.class, Integer.MAX_VALUE, Integer.MAX_VALUE, 0, false, false, false), - OBJECT( JDBCType.STRUCT, null, -1, 0, 0), - NESTED( JDBCType.STRUCT, null, -1, 0, 0), + OBJECT( JDBCType.STRUCT, null, -1, 0, 0, false, false, false), + NESTED( JDBCType.STRUCT, null, -1, 0, 0, false, false, false), BINARY( JDBCType.VARBINARY, byte[].class, -1, Integer.MAX_VALUE, 0), // since ODBC and JDBC interpret precision for Date as display size, // the precision is 23 (number of chars in ISO8601 with millis) + Z (the UTC timezone) @@ -223,7 +223,11 @@ public static DataType fromODBCType(String odbcType) { * For any dataType DataType.fromEsType(dataType.esType) == dataType */ public static DataType fromEsType(String esType) { - return DataType.valueOf(esType.toUpperCase(Locale.ROOT)); + try { + return DataType.valueOf(esType.toUpperCase(Locale.ROOT)); + } catch (IllegalArgumentException ex) { + return DataType.UNSUPPORTED; + } } public boolean isCompatibleWith(DataType other) { diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolver.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolver.java index 0382729aa9f01..574106f07cae4 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolver.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolver.java @@ -15,6 +15,8 @@ import org.elasticsearch.action.admin.indices.get.GetIndexRequest; import org.elasticsearch.action.admin.indices.get.GetIndexRequest.Feature; import org.elasticsearch.action.admin.indices.get.GetIndexResponse; +import org.elasticsearch.action.fieldcaps.FieldCapabilities; +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.IndicesOptions.Option; import org.elasticsearch.action.support.IndicesOptions.WildcardStates; @@ -24,23 +26,34 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.collect.ImmutableOpenMap; import org.elasticsearch.index.IndexNotFoundException; +import org.elasticsearch.xpack.sql.SqlIllegalArgumentException; +import org.elasticsearch.xpack.sql.type.DataType; +import org.elasticsearch.xpack.sql.type.DateEsField; import org.elasticsearch.xpack.sql.type.EsField; +import org.elasticsearch.xpack.sql.type.KeywordEsField; +import org.elasticsearch.xpack.sql.type.TextEsField; import org.elasticsearch.xpack.sql.type.Types; +import org.elasticsearch.xpack.sql.type.UnsupportedEsField; import org.elasticsearch.xpack.sql.util.CollectionUtils; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.EnumSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Map.Entry; +import java.util.NavigableSet; import java.util.Objects; import java.util.Set; +import java.util.TreeMap; import java.util.TreeSet; import java.util.regex.Pattern; -import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; public class IndexResolver { @@ -222,64 +235,157 @@ private void filterResults(String javaRegex, GetAliasesResponse aliases, GetInde listener.onResponse(result); } - /** * Resolves a pattern to one (potentially compound meaning that spawns multiple indices) mapping. */ - public void resolveWithSameMapping(String indexWildcard, String javaRegex, ActionListener listener) { - GetIndexRequest getIndexRequest = createGetIndexRequest(indexWildcard); - client.admin().indices().getIndex(getIndexRequest, ActionListener.wrap(response -> { - ImmutableOpenMap> mappings = response.getMappings(); - - List resolutions; - if (mappings.size() > 0) { - resolutions = new ArrayList<>(mappings.size()); - Pattern pattern = javaRegex != null ? Pattern.compile(javaRegex) : null; - for (ObjectObjectCursor> indexMappings : mappings) { - String concreteIndex = indexMappings.key; - if (pattern == null || pattern.matcher(concreteIndex).matches()) { - resolutions.add(buildGetIndexResult(concreteIndex, concreteIndex, indexMappings.value)); + public void resolveAsMergedMapping(String indexWildcard, String javaRegex, ActionListener listener) { + FieldCapabilitiesRequest fieldRequest = createFieldCapsRequest(indexWildcard); + client.fieldCaps(fieldRequest, + ActionListener.wrap(response -> listener.onResponse(mergedMapping(indexWildcard, response.get())), listener::onFailure)); + } + + static IndexResolution mergedMapping(String indexPattern, Map> fieldCaps) { + if (fieldCaps == null || fieldCaps.isEmpty()) { + return IndexResolution.notFound(indexPattern); + } + + StringBuilder errorMessage = new StringBuilder(); + + NavigableSet>> sortedFields = new TreeSet<>( + // for some reason .reversed doesn't work (prolly due to inference) + Collections.reverseOrder(Comparator.comparing(Entry::getKey))); + sortedFields.addAll(fieldCaps.entrySet()); + + Map hierarchicalMapping = new TreeMap<>(); + Map flattedMapping = new LinkedHashMap<>(); + + // sort keys descending in order to easily detect multi-fields (a.b.c multi-field of a.b) + // without sorting, they can still be detected however without the emptyMap optimization + // (fields without multi-fields have no children) + for (Entry> entry : sortedFields) { + String name = entry.getKey(); + // skip internal fields + if (!name.startsWith("_")) { + Map types = entry.getValue(); + // field is mapped differently across indices + if (types.size() > 1) { + // build error message + for (Entry type : types.entrySet()) { + if (errorMessage.length() > 0) { + errorMessage.append(", "); + } + errorMessage.append("["); + errorMessage.append(type.getKey()); + errorMessage.append("] in "); + errorMessage.append(Arrays.toString(type.getValue().indices())); } + + errorMessage.insert(0, + "[" + indexPattern + "] points to indices with incompatible mappings; " + + "field [" + name + "] is mapped in [" + types.size() + "] different ways: "); + } + if (errorMessage.length() > 0) { + return IndexResolution.invalid(errorMessage.toString()); + } + + FieldCapabilities fieldCap = types.values().iterator().next(); + // validate search/agg-able + if (fieldCap.isAggregatable() && fieldCap.nonAggregatableIndices() != null) { + errorMessage.append("[" + indexPattern + "] points to indices with incompatible mappings: "); + errorMessage.append("field [" + name + "] is aggregateable except in "); + errorMessage.append(Arrays.toString(fieldCap.nonAggregatableIndices())); + } + if (fieldCap.isSearchable() && fieldCap.nonSearchableIndices() != null) { + if (errorMessage.length() > 0) { + errorMessage.append(","); + } + errorMessage.append("[" + indexPattern + "] points to indices with incompatible mappings: "); + errorMessage.append("field [" + name + "] is searchable except in "); + errorMessage.append(Arrays.toString(fieldCap.nonSearchableIndices())); + } + if (errorMessage.length() > 0) { + return IndexResolution.invalid(errorMessage.toString()); + } + + // validation passes - create the field + // and name wasn't added before + if (!flattedMapping.containsKey(name)) { + createField(name, fieldCap, fieldCaps, hierarchicalMapping, flattedMapping, false); } - } else { - resolutions = emptyList(); } + } - listener.onResponse(merge(resolutions, indexWildcard)); - }, listener::onFailure)); + return IndexResolution.valid(new EsIndex(indexPattern, hierarchicalMapping)); } - static IndexResolution merge(List resolutions, String indexWildcard) { - IndexResolution merged = null; - for (IndexResolution resolution : resolutions) { - // everything that follows gets compared - if (!resolution.isValid()) { - return resolution; - } - // initialize resolution on first run - if (merged == null) { - merged = resolution; - } - // need the same mapping across all resolutions - if (!merged.get().mapping().equals(resolution.get().mapping())) { - return IndexResolution.invalid( - "[" + indexWildcard + "] points to indices [" + merged.get().name() + "] " - + "and [" + resolution.get().name() + "] which have different mappings. " - + "When using multiple indices, the mappings must be identical."); + private static EsField createField(String fieldName, FieldCapabilities caps, Map> globalCaps, + Map hierarchicalMapping, Map flattedMapping, boolean hasChildren) { + + Map parentProps = hierarchicalMapping; + + int dot = fieldName.lastIndexOf('.'); + String fullFieldName = fieldName; + + if (dot >= 0) { + String parentName = fieldName.substring(0, dot); + fieldName = fieldName.substring(dot + 1); + EsField parent = flattedMapping.get(parentName); + if (parent == null) { + Map map = globalCaps.get(parentName); + if (map == null) { + throw new SqlIllegalArgumentException("Cannot find field {}; this is likely a bug", parentName); + } + FieldCapabilities parentCap = map.values().iterator().next(); + parent = createField(parentName, parentCap, globalCaps, hierarchicalMapping, flattedMapping, true); } + parentProps = parent.getProperties(); } - if (merged != null) { - // at this point, we are sure there's the same mapping across all (if that's the case) indices - // to keep things simple, use the given pattern as index name - merged = IndexResolution.valid(new EsIndex(indexWildcard, merged.get().mapping())); - } else { - merged = IndexResolution.notFound(indexWildcard); + + EsField field = null; + Map props = hasChildren ? new TreeMap<>() : emptyMap(); + + DataType esType = DataType.fromEsType(caps.getType()); + switch (esType) { + case TEXT: + field = new TextEsField(fieldName, props, false); + break; + case KEYWORD: + int length = DataType.KEYWORD.defaultPrecision; + // TODO: to check whether isSearchable/isAggregateable takes into account the presence of the normalizer + boolean normalized = false; + field = new KeywordEsField(fieldName, props, caps.isAggregatable(), length, normalized); + break; + case DATE: + field = new DateEsField(fieldName, props, caps.isAggregatable()); + break; + case UNSUPPORTED: + field = new UnsupportedEsField(fieldName, caps.getType()); + break; + default: + field = new EsField(fieldName, esType, props, caps.isAggregatable()); } - return merged; + + parentProps.put(fieldName, field); + flattedMapping.put(fullFieldName, field); + + return field; + } + + private static FieldCapabilitiesRequest createFieldCapsRequest(String index) { + return new FieldCapabilitiesRequest() + .indices(Strings.commaDelimitedListToStringArray(index)) + .fields("*") + //lenient because we throw our own errors looking at the response e.g. if something was not resolved + //also because this way security doesn't throw authorization exceptions but rather honors ignore_unavailable + .indicesOptions(IndicesOptions.lenientExpandOpen()); } + // TODO: Concrete indices still uses get mapping + // waiting on https://github.com/elastic/elasticsearch/pull/34071 + // + /** - * Resolves a pattern to multiple, separate indices. + * Resolves a pattern to multiple, separate indices. Doesn't perform validation. */ public void resolveAsSeparateMappings(String indexWildcard, String javaRegex, ActionListener> listener) { GetIndexRequest getIndexRequest = createGetIndexRequest(indexWildcard); @@ -306,7 +412,7 @@ public void resolveAsSeparateMappings(String indexWildcard, String javaRegex, Ac listener.onResponse(results); }, listener::onFailure)); } - + private static GetIndexRequest createGetIndexRequest(String index) { return new GetIndexRequest() .local(true) diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java index aa2e784de3d28..24f55b1d8eb54 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/plan/logical/command/ShowColumns.java @@ -54,13 +54,15 @@ protected NodeInfo info() { @Override public List output() { return asList(new FieldAttribute(location(), "column", new KeywordEsField("column")), - new FieldAttribute(location(), "type", new KeywordEsField("type"))); } + new FieldAttribute(location(), "type", new KeywordEsField("type")), + new FieldAttribute(location(), "mapping", new KeywordEsField("mapping"))); + } @Override public void execute(SqlSession session, ActionListener listener) { String idx = index != null ? index : (pattern != null ? pattern.asIndexNameWildcard() : "*"); String regex = pattern != null ? pattern.asJavaRegex() : null; - session.indexResolver().resolveWithSameMapping(idx, regex, ActionListener.wrap( + session.indexResolver().resolveAsMergedMapping(idx, regex, ActionListener.wrap( indexResult -> { List> rows = emptyList(); if (indexResult.isValid()) { @@ -69,8 +71,7 @@ public void execute(SqlSession session, ActionListener listener) { } listener.onResponse(Rows.of(output(), rows)); }, - listener::onFailure - )); + listener::onFailure)); } private void fillInRows(Map mapping, String prefix, List> rows) { @@ -79,7 +80,7 @@ private void fillInRows(Map mapping, String prefix, List nestedHitFieldRef(FieldAttribute attr) { @@ -181,10 +181,10 @@ private Tuple nestedHitFieldRef(FieldAttribute String name = aliasName(attr); Query q = rewriteToContainNestedField(query, attr.location(), - attr.nestedParent().name(), name, attr.field().hasDocValues()); + attr.nestedParent().name(), name, attr.field().isAggregatable()); SearchHitFieldRef nestedFieldRef = new SearchHitFieldRef(name, attr.field().getDataType(), - attr.field().hasDocValues(), attr.parent().name()); + attr.field().isAggregatable(), attr.parent().name()); nestedRefs.add(nestedFieldRef); return new Tuple<>(new QueryContainer(q, aggs, columns, aliases, pseudoFunctions, scalarFunctions, sort, limit), nestedFieldRef); diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java index 65da32c3122ab..9b31d069cbe79 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/session/SqlSession.java @@ -127,7 +127,7 @@ private void preAnalyze(LogicalPlan parsed, Function act listener.onFailure(new MappingException("Cannot inspect indices in cluster/catalog [{}]", cluster)); } - indexResolver.resolveWithSameMapping(table.index(), null, + indexResolver.resolveAsMergedMapping(table.index(), null, wrap(indexResult -> listener.onResponse(action.apply(indexResult)), listener::onFailure)); } else { try { diff --git a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/EsField.java b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/EsField.java index cc7e085416caa..5630c9409af95 100644 --- a/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/EsField.java +++ b/x-pack/plugin/sql/src/main/java/org/elasticsearch/xpack/sql/type/EsField.java @@ -15,14 +15,14 @@ */ public class EsField { private final DataType esDataType; - private final boolean hasDocValues; + private final boolean aggregatable; private final Map properties; private final String name; - public EsField(String name, DataType esDataType, Map properties, boolean hasDocValues) { + public EsField(String name, DataType esDataType, Map properties, boolean aggregatable) { this.name = name; this.esDataType = esDataType; - this.hasDocValues = hasDocValues; + this.aggregatable = aggregatable; this.properties = properties; } @@ -41,10 +41,10 @@ public DataType getDataType() { } /** - * The field supports doc values + * This field can be aggregated */ - public boolean hasDocValues() { - return hasDocValues; + public boolean isAggregatable() { + return aggregatable; } /** @@ -85,19 +85,27 @@ public boolean isExact() { return true; } + @Override + public String toString() { + return name + "@" + esDataType.name() + "=" + properties; + } + @Override public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } EsField field = (EsField) o; - return hasDocValues == field.hasDocValues && - esDataType == field.esDataType && - Objects.equals(properties, field.properties) && - Objects.equals(name, field.name); + return aggregatable == field.aggregatable && esDataType == field.esDataType + && Objects.equals(name, field.name) + && Objects.equals(properties, field.properties); } @Override public int hashCode() { - return Objects.hash(esDataType, hasDocValues, properties, name); + return Objects.hash(esDataType, aggregatable, properties, name); } -} +} \ No newline at end of file diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java index 639356b2997f9..89f872714024d 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/analysis/index/IndexResolverTests.java @@ -5,12 +5,18 @@ */ package org.elasticsearch.xpack.sql.analysis.index; +import org.elasticsearch.action.fieldcaps.FieldCapabilities; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.sql.type.DataType; import org.elasticsearch.xpack.sql.type.EsField; import org.elasticsearch.xpack.sql.type.TypesTests; -import java.util.Arrays; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.Map.Entry; public class IndexResolverTests extends ESTestCase { @@ -21,40 +27,175 @@ public void testMergeSameMapping() throws Exception { assertEquals(oneMapping, sameMapping); String wildcard = "*"; - IndexResolution resolution = IndexResolver.merge( - Arrays.asList(IndexResolution.valid(new EsIndex("a", oneMapping)), IndexResolution.valid(new EsIndex("b", sameMapping))), - wildcard); + + IndexResolution resolution = IndexResolver.mergedMapping(wildcard, fromMappings( + new EsIndex("a", oneMapping), + new EsIndex("b", sameMapping))); assertTrue(resolution.isValid()); + assertEqualsMaps(oneMapping, resolution.get().mapping()); + } + + public void testMergeCompatibleMapping() throws Exception { + Map basicMapping = TypesTests.loadMapping("mapping-basic.json", true); + Map numericMapping = TypesTests.loadMapping("mapping-numeric.json", true); + + assertNotEquals(basicMapping, numericMapping); - EsIndex esIndex = resolution.get(); + String wildcard = "*"; + IndexResolution resolution = IndexResolver.mergedMapping(wildcard, fromMappings( + new EsIndex("basic", basicMapping), + new EsIndex("numeric", numericMapping))); - assertEquals(wildcard, esIndex.name()); - assertEquals(sameMapping, esIndex.mapping()); + assertTrue(resolution.isValid()); + assertEquals(basicMapping.size() + numericMapping.size(), resolution.get().mapping().size()); } - public void testMergeDifferentMapping() throws Exception { - Map oneMapping = TypesTests.loadMapping("mapping-basic.json", true); - Map sameMapping = TypesTests.loadMapping("mapping-basic.json", true); - Map differentMapping = TypesTests.loadMapping("mapping-numeric.json", true); + public void testMergeIncompatibleTypes() throws Exception { + Map basicMapping = TypesTests.loadMapping("mapping-basic.json", true); + Map incompatible = TypesTests.loadMapping("mapping-basic-incompatible.json"); - assertNotSame(oneMapping, sameMapping); - assertEquals(oneMapping, sameMapping); - assertNotEquals(oneMapping, differentMapping); + assertNotEquals(basicMapping, incompatible); String wildcard = "*"; - IndexResolution resolution = IndexResolver.merge( - Arrays.asList(IndexResolution.valid(new EsIndex("a", oneMapping)), - IndexResolution.valid(new EsIndex("b", sameMapping)), - IndexResolution.valid(new EsIndex("diff", differentMapping))), - wildcard); + IndexResolution resolution = IndexResolver.mergedMapping(wildcard, + fromMappings(new EsIndex("basic", basicMapping), new EsIndex("incompatible", incompatible))); assertFalse(resolution.isValid()); + MappingException me = expectThrows(MappingException.class, () -> resolution.get()); + assertEquals( + "[*] points to indices with incompatible mappings;" + + " field [gender] is mapped in [2] different ways: [text] in [incompatible], [keyword] in [basic]", + me.getMessage()); + } + + public void testMergeIncompatibleCapabilities() throws Exception { + Map basicMapping = TypesTests.loadMapping("mapping-basic.json", true); + Map incompatible = TypesTests.loadMapping("mapping-basic-nodocvalues.json", true); - MappingException ex = expectThrows(MappingException.class, () -> resolution.get()); + assertNotEquals(basicMapping, incompatible); + + String wildcard = "*"; + IndexResolution resolution = IndexResolver.mergedMapping(wildcard, + fromMappings(new EsIndex("basic", basicMapping), new EsIndex("incompatible", incompatible))); + + assertFalse(resolution.isValid()); + MappingException me = expectThrows(MappingException.class, () -> resolution.get()); assertEquals( - "[*] points to indices [a] and [diff] which have different mappings. " - + "When using multiple indices, the mappings must be identical.", - ex.getMessage()); + "[*] points to indices with incompatible mappings: field [emp_no] is aggregateable except in [incompatible]", + me.getMessage()); + } + + public void testMultiLevelObjectMappings() throws Exception { + Map dottedMapping = TypesTests.loadMapping("mapping-dotted-field.json", true); + + String wildcard = "*"; + IndexResolution resolution = IndexResolver.mergedMapping(wildcard, fromMappings(new EsIndex("a", dottedMapping))); + + assertTrue(resolution.isValid()); + assertEqualsMaps(dottedMapping, resolution.get().mapping()); + } + + public void testMultiLevelNestedMappings() throws Exception { + Map nestedMapping = TypesTests.loadMapping("mapping-nested.json", true); + + String wildcard = "*"; + IndexResolution resolution = IndexResolver.mergedMapping(wildcard, fromMappings(new EsIndex("a", nestedMapping))); + + assertTrue(resolution.isValid()); + assertEqualsMaps(nestedMapping, resolution.get().mapping()); + } + + private static Map> fromMappings(EsIndex... indices) { + Map> merged = new HashMap<>(); + + // first pass: create the field caps + for (EsIndex index : indices) { + for (EsField field : index.mapping().values()) { + addFieldCaps(null, field, index.name(), merged); + } + } + + // second pass: update indices + for (Entry> entry : merged.entrySet()) { + String fieldName = entry.getKey(); + Map caps = entry.getValue(); + if (entry.getValue().size() > 1) { + for (EsIndex index : indices) { + EsField field = index.mapping().get(fieldName); + UpdateableFieldCapabilities fieldCaps = (UpdateableFieldCapabilities) caps.get(field.getDataType().esType); + fieldCaps.indices.add(index.name()); + } + //TODO: what about nonAgg/SearchIndices? + } + } + + return merged; + } + + private static void addFieldCaps(String parent, EsField field, String indexName, Map> merged) { + String fieldName = parent != null ? parent + "." + field.getName() : field.getName(); + Map map = merged.get(fieldName); + if (map == null) { + map = new HashMap<>(); + merged.put(fieldName, map); + } + FieldCapabilities caps = map.computeIfAbsent(field.getDataType().esType, + esType -> new UpdateableFieldCapabilities(fieldName, esType, + isSearchable(field.getDataType()), + isAggregatable(field.getDataType()))); + + if (!field.isAggregatable()) { + ((UpdateableFieldCapabilities) caps).nonAggregatableIndices.add(indexName); + } + + for (EsField nested : field.getProperties().values()) { + addFieldCaps(fieldName, nested, indexName, merged); + } + } + + private static boolean isSearchable(DataType type) { + return type.isPrimitive(); + } + + private static boolean isAggregatable(DataType type) { + return type.isNumeric() || type == DataType.KEYWORD || type == DataType.DATE; + } + + private static class UpdateableFieldCapabilities extends FieldCapabilities { + List indices = new ArrayList<>(); + List nonSearchableIndices = new ArrayList<>(); + List nonAggregatableIndices = new ArrayList<>(); + + UpdateableFieldCapabilities(String name, String type, boolean isSearchable, boolean isAggregatable) { + super(name, type, isSearchable, isAggregatable); + } + + @Override + public String[] indices() { + return indices.isEmpty() ? null : indices.toArray(new String[indices.size()]); + } + + @Override + public String[] nonSearchableIndices() { + return nonSearchableIndices.isEmpty() ? null : nonSearchableIndices.toArray(new String[nonSearchableIndices.size()]); + } + + @Override + public String[] nonAggregatableIndices() { + return nonAggregatableIndices.isEmpty() ? null : nonAggregatableIndices.toArray(new String[nonAggregatableIndices.size()]); + } + + @Override + public String toString() { + return String.format(Locale.ROOT, "%s,%s->%s", getName(), getType(), indices); + } + } + + private static void assertEqualsMaps(Map left, Map right) { + for (Entry entry : left.entrySet()) { + V rv = right.get(entry.getKey()); + assertEquals(String.format(Locale.ROOT, "Key [%s] has different values", entry.getKey()), entry.getValue(), rv); + } } -} +} \ No newline at end of file diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/TypesTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/TypesTests.java index 891b11ba70bb0..30f9d82ff77af 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/TypesTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/type/TypesTests.java @@ -59,10 +59,10 @@ public void testTextField() { assertThat(mapping.size(), is(1)); EsField type = mapping.get("full_name"); assertThat(type, instanceOf(TextEsField.class)); - assertThat(type.hasDocValues(), is(false)); + assertThat(type.isAggregatable(), is(false)); TextEsField ttype = (TextEsField) type; assertThat(type.getPrecision(), is(Integer.MAX_VALUE)); - assertThat(ttype.hasDocValues(), is(false)); + assertThat(ttype.isAggregatable(), is(false)); } public void testKeywordField() { @@ -71,7 +71,7 @@ public void testKeywordField() { assertThat(mapping.size(), is(1)); EsField field = mapping.get("full_name"); assertThat(field, instanceOf(KeywordEsField.class)); - assertThat(field.hasDocValues(), is(true)); + assertThat(field.isAggregatable(), is(true)); assertThat(field.getPrecision(), is(256)); } @@ -81,7 +81,7 @@ public void testDateField() { assertThat(mapping.size(), is(1)); EsField field = mapping.get("date"); assertThat(field.getDataType(), is(DATE)); - assertThat(field.hasDocValues(), is(true)); + assertThat(field.isAggregatable(), is(true)); assertThat(field.getPrecision(), is(24)); DateEsField dfield = (DateEsField) field; @@ -95,7 +95,7 @@ public void testDateNoFormat() { assertThat(mapping.size(), is(1)); EsField field = mapping.get("date"); assertThat(field.getDataType(), is(DATE)); - assertThat(field.hasDocValues(), is(true)); + assertThat(field.isAggregatable(), is(true)); DateEsField dfield = (DateEsField) field; // default types assertThat(dfield.getFormats(), hasSize(2)); @@ -107,7 +107,7 @@ public void testDateMulti() { assertThat(mapping.size(), is(1)); EsField field = mapping.get("date"); assertThat(field.getDataType(), is(DATE)); - assertThat(field.hasDocValues(), is(true)); + assertThat(field.isAggregatable(), is(true)); DateEsField dfield = (DateEsField) field; // default types assertThat(dfield.getFormats(), hasSize(1)); @@ -120,7 +120,7 @@ public void testDocValueField() { EsField field = mapping.get("session_id"); assertThat(field, instanceOf(KeywordEsField.class)); assertThat(field.getPrecision(), is(15)); - assertThat(field.hasDocValues(), is(false)); + assertThat(field.isAggregatable(), is(false)); } public void testDottedField() { diff --git a/x-pack/plugin/sql/src/test/resources/mapping-basic-incompatible.json b/x-pack/plugin/sql/src/test/resources/mapping-basic-incompatible.json new file mode 100644 index 0000000000000..9042415a51599 --- /dev/null +++ b/x-pack/plugin/sql/src/test/resources/mapping-basic-incompatible.json @@ -0,0 +1,22 @@ +{ + "properties" : { + "emp_no" : { + "type" : "long" + }, + "first_name" : { + "type" : "text" + }, + "gender" : { + "type" : "text" + }, + "languages" : { + "type" : "byte" + }, + "last_name" : { + "type" : "text" + }, + "salary" : { + "type" : "integer" + } + } +} diff --git a/x-pack/plugin/sql/src/test/resources/mapping-basic-nodocvalues.json b/x-pack/plugin/sql/src/test/resources/mapping-basic-nodocvalues.json new file mode 100644 index 0000000000000..bb9cd60dc02e0 --- /dev/null +++ b/x-pack/plugin/sql/src/test/resources/mapping-basic-nodocvalues.json @@ -0,0 +1,23 @@ +{ + "properties" : { + "emp_no" : { + "type" : "integer", + "doc_values" : false + }, + "first_name" : { + "type" : "text" + }, + "gender" : { + "type" : "keyword" + }, + "languages" : { + "type" : "byte" + }, + "last_name" : { + "type" : "text" + }, + "salary" : { + "type" : "integer" + } + } +} diff --git a/x-pack/qa/sql/security/roles.yml b/x-pack/qa/sql/security/roles.yml index 1759c972d34bf..337d7c7f9c7c1 100644 --- a/x-pack/qa/sql/security/roles.yml +++ b/x-pack/qa/sql/security/roles.yml @@ -7,8 +7,8 @@ rest_minimal: privileges: [read, "indices:admin/get"] # end::rest -# tag::cli_jdbc -cli_or_jdbc_minimal: +# tag::cli_drivers +cli_or_drivers_minimal: cluster: - "cluster:monitor/main" indices: @@ -16,7 +16,7 @@ cli_or_jdbc_minimal: privileges: [read, "indices:admin/get"] - names: bort privileges: [read, "indices:admin/get"] -# end::cli_jdbc +# end::cli_drivers read_something_else: cluster: @@ -82,6 +82,6 @@ no_get_index: - "cluster:monitor/main" indices: - names: test - privileges: [read] + privileges: [monitor] - names: bort - privileges: [read] + privileges: [monitor] diff --git a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/CliSecurityIT.java b/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/CliSecurityIT.java index 5e8aa4ec6ad8b..3e0b578913841 100644 --- a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/CliSecurityIT.java +++ b/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/CliSecurityIT.java @@ -7,9 +7,10 @@ import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.io.PathUtils; -import org.elasticsearch.xpack.qa.sql.cli.ErrorsTestCase; import org.elasticsearch.xpack.qa.sql.cli.EmbeddedCli; import org.elasticsearch.xpack.qa.sql.cli.EmbeddedCli.SecurityConfig; +import org.elasticsearch.xpack.qa.sql.cli.ErrorsTestCase; + import java.io.IOException; import java.net.URISyntaxException; import java.nio.file.Files; @@ -20,7 +21,6 @@ import java.util.Map; import static org.elasticsearch.xpack.qa.sql.cli.CliIntegrationTestCase.elasticsearchAddress; -import static org.hamcrest.Matchers.both; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.startsWith; @@ -53,7 +53,7 @@ static SecurityConfig adminSecurityConfig() { private static class CliActions implements Actions { @Override public String minimalPermissionsForAllActions() { - return "cli_or_jdbc_minimal"; + return "cli_or_drivers_minimal"; } private SecurityConfig userSecurity(String user) { @@ -121,12 +121,19 @@ public void expectMatchesAdmin(String adminSql, String user, String userSql, } @Override - public void expectDescribe(Map columns, String user) throws Exception { + public void expectDescribe(Map> columns, String user) throws Exception { try (EmbeddedCli cli = new EmbeddedCli(elasticsearchAddress(), true, userSecurity(user))) { - assertThat(cli.command("DESCRIBE test"), containsString("column | type")); - assertEquals("---------------+---------------", cli.readLine()); - for (Map.Entry column : columns.entrySet()) { - assertThat(cli.readLine(), both(startsWith(column.getKey())).and(containsString("|" + column.getValue()))); + String output = cli.command("DESCRIBE test"); + assertThat(output, containsString("column")); + assertThat(output, containsString("type")); + assertThat(output, containsString("mapping")); + assertThat(cli.readLine(), containsString("-+---------------+---------------")); + for (Map.Entry> column : columns.entrySet()) { + String line = cli.readLine(); + assertThat(line, startsWith(column.getKey())); + for (String value : column.getValue()) { + assertThat(line, containsString(value)); + } } assertEquals("", cli.readLine()); } diff --git a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/JdbcSecurityIT.java b/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/JdbcSecurityIT.java index 48b850d0acf21..848b98eeb7bf8 100644 --- a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/JdbcSecurityIT.java +++ b/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/JdbcSecurityIT.java @@ -118,7 +118,7 @@ static void expectActionThrowsUnknownColumn(String user, private static class JdbcActions implements Actions { @Override public String minimalPermissionsForAllActions() { - return "cli_or_jdbc_minimal"; + return "cli_or_drivers_minimal"; } @Override @@ -158,22 +158,26 @@ public void expectScrollMatchesAdmin(String adminSql, String user, String userSq } @Override - public void expectDescribe(Map columns, String user) throws Exception { + public void expectDescribe(Map> columns, String user) throws Exception { try (Connection h2 = LocalH2.anonymousDb(); Connection es = es(userProperties(user))) { // h2 doesn't have the same sort of DESCRIBE that we have so we emulate it - h2.createStatement().executeUpdate("CREATE TABLE mock (column VARCHAR, type VARCHAR)"); + h2.createStatement().executeUpdate("CREATE TABLE mock (column VARCHAR, type VARCHAR, mapping VARCHAR)"); if (columns.size() > 0) { StringBuilder insert = new StringBuilder(); - insert.append("INSERT INTO mock (column, type) VALUES "); + insert.append("INSERT INTO mock (column, type, mapping) VALUES "); boolean first = true; - for (Map.Entry column : columns.entrySet()) { + for (Map.Entry> column : columns.entrySet()) { if (first) { first = false; } else { insert.append(", "); } - insert.append("('").append(column.getKey()).append("', '").append(column.getValue()).append("')"); + insert.append("('").append(column.getKey()).append("'"); + for (String value : column.getValue()) { + insert.append(", '").append(value).append("'"); + } + insert.append(")"); } h2.createStatement().executeUpdate(insert.toString()); } @@ -250,7 +254,7 @@ public JdbcSecurityIT() { // Metadata methods only available to JDBC public void testMetaDataGetTablesWithFullAccess() throws Exception { - createUser("full_access", "cli_or_jdbc_minimal"); + createUser("full_access", "cli_or_drivers_minimal"); expectActionMatchesAdmin( con -> con.getMetaData().getTables("%", "%", "%t", null), @@ -283,7 +287,7 @@ public void testMetaDataGetTablesWithInAccessibleIndex() throws Exception { } public void testMetaDataGetColumnsWorksAsFullAccess() throws Exception { - createUser("full_access", "cli_or_jdbc_minimal"); + createUser("full_access", "cli_or_drivers_minimal"); expectActionMatchesAdmin( con -> con.getMetaData().getColumns(null, "%", "%t", "%"), diff --git a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/RestSqlSecurityIT.java b/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/RestSqlSecurityIT.java index 51440cc68dd0b..607b37e39d192 100644 --- a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/RestSqlSecurityIT.java +++ b/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/RestSqlSecurityIT.java @@ -93,15 +93,19 @@ public void expectScrollMatchesAdmin(String adminSql, String user, String userSq } @Override - public void expectDescribe(Map columns, String user) throws Exception { + public void expectDescribe(Map> columns, String user) throws Exception { String mode = randomMode(); Map expected = new HashMap<>(3); expected.put("columns", Arrays.asList( columnInfo(mode, "column", "keyword", JDBCType.VARCHAR, 0), - columnInfo(mode, "type", "keyword", JDBCType.VARCHAR, 0))); + columnInfo(mode, "type", "keyword", JDBCType.VARCHAR, 0), + columnInfo(mode, "mapping", "keyword", JDBCType.VARCHAR, 0))); List> rows = new ArrayList<>(columns.size()); - for (Map.Entry column : columns.entrySet()) { - rows.add(Arrays.asList(column.getKey(), column.getValue())); + for (Map.Entry> column : columns.entrySet()) { + List cols = new ArrayList<>(); + cols.add(column.getKey()); + cols.addAll(column.getValue()); + rows.add(cols); } expected.put("rows", rows); @@ -232,7 +236,7 @@ public void testHijackScrollFails() throws Exception { assertEquals(404, e.getResponse().getStatusLine().getStatusCode()); createAuditLogAsserter() - .expectSqlCompositeAction("test_admin", "test") + .expectSqlCompositeActionFieldCaps("test_admin", "test") .expect(true, SQL_ACTION_NAME, "full_access", empty()) // one scroll access denied per shard .expect("access_denied", SQL_ACTION_NAME, "full_access", "default_native", empty(), "InternalScrollSearchRequest") diff --git a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/SqlSecurityTestCase.java b/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/SqlSecurityTestCase.java index c9076e38a0d4f..0e3d2cab2ff84 100644 --- a/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/SqlSecurityTestCase.java +++ b/x-pack/qa/sql/security/src/test/java/org/elasticsearch/xpack/qa/sql/security/SqlSecurityTestCase.java @@ -10,6 +10,8 @@ import org.elasticsearch.SpecialPermission; import org.elasticsearch.action.admin.indices.get.GetIndexAction; import org.elasticsearch.action.admin.indices.get.GetIndexRequest; +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesAction; +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesRequest; import org.elasticsearch.client.Request; import org.elasticsearch.client.ResponseException; import org.elasticsearch.common.Strings; @@ -40,11 +42,12 @@ import java.util.TreeMap; import java.util.function.Function; +import static java.util.Arrays.asList; import static java.util.Collections.singletonMap; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.hasItems; +import static org.hamcrest.Matchers.is; public abstract class SqlSecurityTestCase extends ESRestTestCase { /** @@ -65,7 +68,7 @@ protected interface Actions { * to 1 and completely scrolls the results. */ void expectScrollMatchesAdmin(String adminSql, String user, String userSql) throws Exception; - void expectDescribe(Map columns, String user) throws Exception; + void expectDescribe(Map> columns, String user) throws Exception; void expectShowTables(List tables, String user) throws Exception; void expectForbidden(String user, String sql) throws Exception; void expectUnknownIndex(String user, String sql) throws Exception; @@ -196,7 +199,7 @@ protected String getProtocol() { public void testQueryWorksAsAdmin() throws Exception { actions.queryWorksAsAdmin(); createAuditLogAsserter() - .expectSqlCompositeAction("test_admin", "test") + .expectSqlCompositeActionFieldCaps("test_admin", "test") .assertLogs(); } @@ -205,8 +208,8 @@ public void testQueryWithFullAccess() throws Exception { actions.expectMatchesAdmin("SELECT * FROM test ORDER BY a", "full_access", "SELECT * FROM test ORDER BY a"); createAuditLogAsserter() - .expectSqlCompositeAction("test_admin", "test") - .expectSqlCompositeAction("full_access", "test") + .expectSqlCompositeActionFieldCaps("test_admin", "test") + .expectSqlCompositeActionFieldCaps("full_access", "test") .assertLogs(); } @@ -215,12 +218,12 @@ public void testScrollWithFullAccess() throws Exception { actions.expectScrollMatchesAdmin("SELECT * FROM test ORDER BY a", "full_access", "SELECT * FROM test ORDER BY a"); createAuditLogAsserter() - .expectSqlCompositeAction("test_admin", "test") + .expectSqlCompositeActionFieldCaps("test_admin", "test") /* Scrolling doesn't have to access the index again, at least not through sql. * If we asserted query and scroll logs then we would see the scroll. */ .expect(true, SQL_ACTION_NAME, "test_admin", empty()) .expect(true, SQL_ACTION_NAME, "test_admin", empty()) - .expectSqlCompositeAction("full_access", "test") + .expectSqlCompositeActionFieldCaps("full_access", "test") .expect(true, SQL_ACTION_NAME, "full_access", empty()) .expect(true, SQL_ACTION_NAME, "full_access", empty()) .assertLogs(); @@ -243,7 +246,7 @@ public void testQueryWrongAccess() throws Exception { //This user has permission to run sql queries so they are given preliminary authorization .expect(true, SQL_ACTION_NAME, "wrong_access", empty()) //the following get index is granted too but against the no indices placeholder, as ignore_unavailable=true - .expect(true, GetIndexAction.NAME, "wrong_access", hasItems("*", "-*")) + .expect(true, FieldCapabilitiesAction.NAME, "wrong_access", hasItems("*", "-*")) .assertLogs(); } @@ -252,8 +255,8 @@ public void testQuerySingleFieldGranted() throws Exception { actions.expectMatchesAdmin("SELECT a FROM test ORDER BY a", "only_a", "SELECT * FROM test ORDER BY a"); createAuditLogAsserter() - .expectSqlCompositeAction("test_admin", "test") - .expectSqlCompositeAction("only_a", "test") + .expectSqlCompositeActionFieldCaps("test_admin", "test") + .expectSqlCompositeActionFieldCaps("only_a", "test") .assertLogs(); } @@ -262,18 +265,18 @@ public void testScrollWithSingleFieldGranted() throws Exception { actions.expectScrollMatchesAdmin("SELECT a FROM test ORDER BY a", "only_a", "SELECT * FROM test ORDER BY a"); createAuditLogAsserter() - .expectSqlCompositeAction("test_admin", "test") + .expectSqlCompositeActionFieldCaps("test_admin", "test") /* Scrolling doesn't have to access the index again, at least not through sql. * If we asserted query and scroll logs then we would see the scroll. */ .expect(true, SQL_ACTION_NAME, "test_admin", empty()) .expect(true, SQL_ACTION_NAME, "test_admin", empty()) - .expectSqlCompositeAction("only_a", "test") + .expectSqlCompositeActionFieldCaps("only_a", "test") .expect(true, SQL_ACTION_NAME, "only_a", empty()) .expect(true, SQL_ACTION_NAME, "only_a", empty()) .assertLogs(); } - public void testQueryStringSingeFieldGrantedWrongRequested() throws Exception { + public void testQueryStringSingleFieldGrantedWrongRequested() throws Exception { createUser("only_a", "read_test_a"); actions.expectUnknownColumn("only_a", "SELECT c FROM test", "c"); @@ -284,7 +287,7 @@ public void testQueryStringSingeFieldGrantedWrongRequested() throws Exception { * out but it failed in SQL because it couldn't compile the * query without the metadata for the missing field. */ createAuditLogAsserter() - .expectSqlCompositeAction("only_a", "test") + .expectSqlCompositeActionFieldCaps("only_a", "test") .assertLogs(); } @@ -293,8 +296,8 @@ public void testQuerySingleFieldExcepted() throws Exception { actions.expectMatchesAdmin("SELECT a, b FROM test ORDER BY a", "not_c", "SELECT * FROM test ORDER BY a"); createAuditLogAsserter() - .expectSqlCompositeAction("test_admin", "test") - .expectSqlCompositeAction("not_c", "test") + .expectSqlCompositeActionFieldCaps("test_admin", "test") + .expectSqlCompositeActionFieldCaps("not_c", "test") .assertLogs(); } @@ -303,12 +306,12 @@ public void testScrollWithSingleFieldExcepted() throws Exception { actions.expectScrollMatchesAdmin("SELECT a, b FROM test ORDER BY a", "not_c", "SELECT * FROM test ORDER BY a"); createAuditLogAsserter() - .expectSqlCompositeAction("test_admin", "test") + .expectSqlCompositeActionFieldCaps("test_admin", "test") /* Scrolling doesn't have to access the index again, at least not through sql. * If we asserted query and scroll logs then we would see the scroll. */ .expect(true, SQL_ACTION_NAME, "test_admin", empty()) .expect(true, SQL_ACTION_NAME, "test_admin", empty()) - .expectSqlCompositeAction("not_c", "test") + .expectSqlCompositeActionFieldCaps("not_c", "test") .expect(true, SQL_ACTION_NAME, "not_c", empty()) .expect(true, SQL_ACTION_NAME, "not_c", empty()) .assertLogs(); @@ -325,7 +328,7 @@ public void testQuerySingleFieldExceptionedWrongRequested() throws Exception { * out but it failed in SQL because it couldn't compile the * query without the metadata for the missing field. */ createAuditLogAsserter() - .expectSqlCompositeAction("not_c", "test") + .expectSqlCompositeActionFieldCaps("not_c", "test") .assertLogs(); } @@ -334,15 +337,15 @@ public void testQueryDocumentExcluded() throws Exception { actions.expectMatchesAdmin("SELECT * FROM test WHERE c != 3 ORDER BY a", "no_3s", "SELECT * FROM test ORDER BY a"); createAuditLogAsserter() - .expectSqlCompositeAction("test_admin", "test") - .expectSqlCompositeAction("no_3s", "test") + .expectSqlCompositeActionFieldCaps("test_admin", "test") + .expectSqlCompositeActionFieldCaps("no_3s", "test") .assertLogs(); } public void testShowTablesWorksAsAdmin() throws Exception { actions.expectShowTables(Arrays.asList("bort", "test"), null); createAuditLogAsserter() - .expectSqlCompositeAction("test_admin", "bort", "test") + .expectSqlCompositeActionGetIndex("test_admin", "bort", "test") .assertLogs(); } @@ -351,8 +354,8 @@ public void testShowTablesWorksAsFullAccess() throws Exception { actions.expectMatchesAdmin("SHOW TABLES LIKE '%t'", "full_access", "SHOW TABLES"); createAuditLogAsserter() - .expectSqlCompositeAction("test_admin", "bort", "test") - .expectSqlCompositeAction("full_access", "bort", "test") + .expectSqlCompositeActionGetIndex("test_admin", "bort", "test") + .expectSqlCompositeActionGetIndex("full_access", "bort", "test") .assertLogs(); } @@ -370,8 +373,7 @@ public void testShowTablesWithLimitedAccess() throws Exception { actions.expectMatchesAdmin("SHOW TABLES LIKE 'bort'", "read_bort", "SHOW TABLES"); createAuditLogAsserter() - .expectSqlCompositeAction("test_admin", "bort") - .expectSqlCompositeAction("read_bort", "bort") + .expectSqlCompositeActionGetIndex("test_admin", "bort").expectSqlCompositeActionGetIndex("read_bort", "bort") .assertLogs(); } @@ -388,13 +390,13 @@ public void testShowTablesWithLimitedAccessUnaccessableIndex() throws Exception } public void testDescribeWorksAsAdmin() throws Exception { - Map expected = new TreeMap<>(); - expected.put("a", "BIGINT"); - expected.put("b", "BIGINT"); - expected.put("c", "BIGINT"); + Map> expected = new TreeMap<>(); + expected.put("a", asList("BIGINT", "LONG")); + expected.put("b", asList("BIGINT", "LONG")); + expected.put("c", asList("BIGINT", "LONG")); actions.expectDescribe(expected, null); createAuditLogAsserter() - .expectSqlCompositeAction("test_admin", "test") + .expectSqlCompositeActionFieldCaps("test_admin", "test") .assertLogs(); } @@ -403,8 +405,8 @@ public void testDescribeWorksAsFullAccess() throws Exception { actions.expectMatchesAdmin("DESCRIBE test", "full_access", "DESCRIBE test"); createAuditLogAsserter() - .expectSqlCompositeAction("test_admin", "test") - .expectSqlCompositeAction("full_access", "test") + .expectSqlCompositeActionFieldCaps("test_admin", "test") + .expectSqlCompositeActionFieldCaps("full_access", "test") .assertLogs(); } @@ -425,28 +427,28 @@ public void testDescribeWithWrongAccess() throws Exception { //This user has permission to run sql queries so they are given preliminary authorization .expect(true, SQL_ACTION_NAME, "wrong_access", empty()) //the following get index is granted too but against the no indices placeholder, as ignore_unavailable=true - .expect(true, GetIndexAction.NAME, "wrong_access", hasItems("*", "-*")) + .expect(true, FieldCapabilitiesAction.NAME, "wrong_access", hasItems("*", "-*")) .assertLogs(); } public void testDescribeSingleFieldGranted() throws Exception { createUser("only_a", "read_test_a"); - actions.expectDescribe(singletonMap("a", "BIGINT"), "only_a"); + actions.expectDescribe(singletonMap("a", asList("BIGINT", "LONG")), "only_a"); createAuditLogAsserter() - .expectSqlCompositeAction("only_a", "test") + .expectSqlCompositeActionFieldCaps("only_a", "test") .assertLogs(); } public void testDescribeSingleFieldExcepted() throws Exception { createUser("not_c", "read_test_a_and_b"); - Map expected = new TreeMap<>(); - expected.put("a", "BIGINT"); - expected.put("b", "BIGINT"); + Map> expected = new TreeMap<>(); + expected.put("a", asList("BIGINT", "LONG")); + expected.put("b", asList("BIGINT", "LONG")); actions.expectDescribe(expected, "not_c"); createAuditLogAsserter() - .expectSqlCompositeAction("not_c", "test") + .expectSqlCompositeActionFieldCaps("not_c", "test") .assertLogs(); } @@ -455,8 +457,8 @@ public void testDescribeDocumentExcluded() throws Exception { actions.expectMatchesAdmin("DESCRIBE test", "no_3s", "DESCRIBE test"); createAuditLogAsserter() - .expectSqlCompositeAction("test_admin", "test") - .expectSqlCompositeAction("no_3s", "test") + .expectSqlCompositeActionFieldCaps("test_admin", "test") + .expectSqlCompositeActionFieldCaps("no_3s", "test") .assertLogs(); } @@ -497,12 +499,18 @@ protected AuditLogAsserter createAuditLogAsserter() { protected class AuditLogAsserter { protected final List, Boolean>> logCheckers = new ArrayList<>(); - public AuditLogAsserter expectSqlCompositeAction(String user, String... indices) { + public AuditLogAsserter expectSqlCompositeActionGetIndex(String user, String... indices) { expect(true, SQL_ACTION_NAME, user, empty()); expect(true, GetIndexAction.NAME, user, hasItems(indices)); return this; } + public AuditLogAsserter expectSqlCompositeActionFieldCaps(String user, String... indices) { + expect(true, SQL_ACTION_NAME, user, empty()); + expect(true, FieldCapabilitiesAction.NAME, user, hasItems(indices)); + return this; + } + public AuditLogAsserter expect(boolean granted, String action, String principal, Matcher> indicesMatcher) { String request; @@ -513,6 +521,9 @@ public AuditLogAsserter expect(boolean granted, String action, String principal, case GetIndexAction.NAME: request = GetIndexRequest.class.getSimpleName(); break; + case FieldCapabilitiesAction.NAME: + request = FieldCapabilitiesRequest.class.getSimpleName(); + break; default: throw new IllegalArgumentException("Unknown action [" + action + "]"); } @@ -523,7 +534,7 @@ public AuditLogAsserter expect(boolean granted, String action, String principal, public AuditLogAsserter expect(String eventAction, String action, String principal, String realm, Matcher> indicesMatcher, String request) { - logCheckers.add(m -> + logCheckers.add(m -> eventAction.equals(m.get("event.action")) && action.equals(m.get("action")) && principal.equals(m.get("user.name")) @@ -564,7 +575,9 @@ public void assertLogs() throws Exception { continue; } assertThat(log.containsKey("action"), is(true)); - if (false == (SQL_ACTION_NAME.equals(log.get("action")) || GetIndexAction.NAME.equals(log.get("action")))) { + if (false == (SQL_ACTION_NAME.equals(log.get("action")) + || GetIndexAction.NAME.equals(log.get("action")) + || FieldCapabilitiesAction.NAME.equals(log.get("action")))) { // TODO we may want to extend this and the assertions to SearchAction.NAME as well continue; } diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/EmbeddedCli.java b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/EmbeddedCli.java index 89184edec0ad7..234d229f324b8 100644 --- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/EmbeddedCli.java +++ b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/EmbeddedCli.java @@ -5,13 +5,13 @@ */ package org.elasticsearch.xpack.qa.sql.cli; +import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.cli.MockTerminal; import org.elasticsearch.cli.Terminal; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Strings; -import org.elasticsearch.common.logging.Loggers; +import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.xpack.sql.cli.Cli; import org.elasticsearch.xpack.sql.cli.CliTerminal; import org.elasticsearch.xpack.sql.cli.JLineTerminal; @@ -49,7 +49,7 @@ * and doesn't run super frequently. */ public class EmbeddedCli implements Closeable { - private static final Logger logger = Loggers.getLogger(EmbeddedCli.class); + private static final Logger logger = LogManager.getLogger(EmbeddedCli.class); private final Thread exec; private final Cli cli; @@ -151,7 +151,9 @@ protected boolean addShutdownHook() { } // Throw out the logo - while (false == readLine().contains("SQL")); + while (false == readLine().contains("SQL")) { + ; + } assertConnectionTest(); } catch (IOException e) { try { diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/ErrorsTestCase.java b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/ErrorsTestCase.java index 18895fbabbc7f..dee99a7be1ca6 100644 --- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/ErrorsTestCase.java +++ b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/cli/ErrorsTestCase.java @@ -5,9 +5,10 @@ */ package org.elasticsearch.xpack.qa.sql.cli; -import java.io.IOException; import org.elasticsearch.client.Request; +import java.io.IOException; + import static org.hamcrest.Matchers.startsWith; /** @@ -43,7 +44,8 @@ public void testSelectFromIndexWithoutTypes() throws Exception { client().performRequest(request); assertFoundOneProblem(command("SELECT * FROM test")); - assertEquals("line 1:15: [test] doesn't have any types so it is incompatible with sql" + END, readLine()); + //assertEquals("line 1:15: [test] doesn't have any types so it is incompatible with sql" + END, readLine()); + assertEquals("line 1:15: Unknown index [test]" + END, readLine()); } @Override diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/DataLoader.java b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/DataLoader.java index 75204d9ff1230..354c44a60eee2 100644 --- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/DataLoader.java +++ b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/DataLoader.java @@ -43,7 +43,7 @@ protected static void loadDatasetIntoEs(RestClient client) throws Exception { protected static void loadEmpDatasetIntoEs(RestClient client) throws Exception { loadEmpDatasetIntoEs(client, "test_emp", "employees"); - loadEmpDatasetIntoEs(client, "test_emp_copy", "employees"); + loadEmpDatasetWithExtraIntoEs(client, "test_emp_copy", "employees"); makeAlias(client, "test_alias", "test_emp", "test_emp_copy"); makeAlias(client, "test_alias_emp", "test_emp", "test_emp_copy"); } @@ -63,6 +63,14 @@ private static void createString(String name, XContentBuilder builder) throws Ex } protected static void loadEmpDatasetIntoEs(RestClient client, String index, String fileName) throws Exception { + loadEmpDatasetIntoEs(client, index, fileName, false); + } + + protected static void loadEmpDatasetWithExtraIntoEs(RestClient client, String index, String fileName) throws Exception { + loadEmpDatasetIntoEs(client, index, fileName, true); + } + + private static void loadEmpDatasetIntoEs(RestClient client, String index, String fileName, boolean extraFields) throws Exception { Request request = new Request("PUT", "/" + index); XContentBuilder createIndex = JsonXContent.contentBuilder().startObject(); createIndex.startObject("settings"); @@ -76,10 +84,26 @@ protected static void loadEmpDatasetIntoEs(RestClient client, String index, Stri { createIndex.startObject("properties"); { - createIndex.startObject("emp_no").field("type", "integer").endObject(); + createIndex.startObject("emp_no").field("type", "integer"); + if (extraFields) { + createIndex.field("copy_to", "extra_no"); + } + createIndex.endObject(); + if (extraFields) { + createIndex.startObject("extra_no").field("type", "integer").endObject(); + } createString("first_name", createIndex); createString("last_name", createIndex); - createIndex.startObject("gender").field("type", "keyword").endObject(); + createIndex.startObject("gender").field("type", "keyword"); + if (extraFields) { + createIndex.field("copy_to", "extra_gender"); + } + createIndex.endObject(); + + if (extraFields) { + createIndex.startObject("extra_gender").field("type", "keyword").endObject(); + } + createIndex.startObject("birth_date").field("type", "date").endObject(); createIndex.startObject("hire_date").field("type", "date").endObject(); createIndex.startObject("salary").field("type", "integer").endObject(); diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/ErrorsTestCase.java b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/ErrorsTestCase.java index 8574dd3ece16e..65fd0778b57f2 100644 --- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/ErrorsTestCase.java +++ b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/ErrorsTestCase.java @@ -5,9 +5,10 @@ */ package org.elasticsearch.xpack.qa.sql.jdbc; +import org.elasticsearch.client.Request; + import java.sql.Connection; import java.sql.SQLException; -import org.elasticsearch.client.Request; import static org.hamcrest.Matchers.startsWith; @@ -40,7 +41,9 @@ public void testSelectFromIndexWithoutTypes() throws Exception { try (Connection c = esJdbc()) { SQLException e = expectThrows(SQLException.class, () -> c.prepareStatement("SELECT * FROM test").executeQuery()); - assertEquals("Found 1 problem(s)\nline 1:15: [test] doesn't have any types so it is incompatible with sql", e.getMessage()); + // see https://github.com/elastic/elasticsearch/issues/34719 + //assertEquals("Found 1 problem(s)\nline 1:15: [test] doesn't have any types so it is incompatible with sql", e.getMessage()); + assertEquals("Found 1 problem(s)\nline 1:15: Unknown index [test]", e.getMessage()); } } diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/rest/RestSqlTestCase.java b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/rest/RestSqlTestCase.java index dd79bd0851491..4df82119e36d0 100644 --- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/rest/RestSqlTestCase.java +++ b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/rest/RestSqlTestCase.java @@ -219,7 +219,9 @@ public void testSelectFromIndexWithoutTypes() throws Exception { client().performRequest(request); String mode = randomFrom("jdbc", "plain"); expectBadRequest(() -> runSql(mode, "SELECT * FROM test"), - containsString("1:15: [test] doesn't have any types so it is incompatible with sql")); + // see https://github.com/elastic/elasticsearch/issues/34719 + //containsString("1:15: [test] doesn't have any types so it is incompatible with sql")); + containsString("1:15: Unknown index [test]")); } @Override diff --git a/x-pack/qa/sql/src/main/resources/alias.csv-spec b/x-pack/qa/sql/src/main/resources/alias.csv-spec index a5f928d73e5a3..2a64bfc34ded3 100644 --- a/x-pack/qa/sql/src/main/resources/alias.csv-spec +++ b/x-pack/qa/sql/src/main/resources/alias.csv-spec @@ -26,47 +26,51 @@ emp_no:i | first_name:s describeAlias DESCRIBE test_alias; -column:s | type:s - -birth_date | TIMESTAMP -dep | STRUCT -dep.dep_id | VARCHAR -dep.dep_name | VARCHAR -dep.dep_name.keyword | VARCHAR -dep.from_date | TIMESTAMP -dep.to_date | TIMESTAMP -emp_no | INTEGER -first_name | VARCHAR -first_name.keyword | VARCHAR -gender | VARCHAR -hire_date | TIMESTAMP -languages | TINYINT -last_name | VARCHAR -last_name.keyword | VARCHAR -salary | INTEGER + column | type | mapping +--------------------+---------------+--------------- +birth_date |TIMESTAMP |DATE +dep |STRUCT |NESTED +dep.dep_id |VARCHAR |KEYWORD +dep.dep_name |VARCHAR |TEXT +dep.dep_name.keyword|VARCHAR |KEYWORD +dep.from_date |TIMESTAMP |DATE +dep.to_date |TIMESTAMP |DATE +emp_no |INTEGER |INTEGER +extra_gender |VARCHAR |KEYWORD +extra_no |INTEGER |INTEGER +first_name |VARCHAR |TEXT +first_name.keyword |VARCHAR |KEYWORD +gender |VARCHAR |KEYWORD +hire_date |TIMESTAMP |DATE +languages |TINYINT |BYTE +last_name |VARCHAR |TEXT +last_name.keyword |VARCHAR |KEYWORD +salary |INTEGER |INTEGER ; describePattern DESCRIBE "test_*"; -column:s | type:s - -birth_date | TIMESTAMP -dep | STRUCT -dep.dep_id | VARCHAR -dep.dep_name | VARCHAR -dep.dep_name.keyword | VARCHAR -dep.from_date | TIMESTAMP -dep.to_date | TIMESTAMP -emp_no | INTEGER -first_name | VARCHAR -first_name.keyword | VARCHAR -gender | VARCHAR -hire_date | TIMESTAMP -languages | TINYINT -last_name | VARCHAR -last_name.keyword | VARCHAR -salary | INTEGER + column | type | mapping +--------------------+---------------+--------------- +birth_date |TIMESTAMP |DATE +dep |STRUCT |NESTED +dep.dep_id |VARCHAR |KEYWORD +dep.dep_name |VARCHAR |TEXT +dep.dep_name.keyword|VARCHAR |KEYWORD +dep.from_date |TIMESTAMP |DATE +dep.to_date |TIMESTAMP |DATE +emp_no |INTEGER |INTEGER +extra_gender |VARCHAR |KEYWORD +extra_no |INTEGER |INTEGER +first_name |VARCHAR |TEXT +first_name.keyword |VARCHAR |KEYWORD +gender |VARCHAR |KEYWORD +hire_date |TIMESTAMP |DATE +languages |TINYINT |BYTE +last_name |VARCHAR |TEXT +last_name.keyword |VARCHAR |KEYWORD +salary |INTEGER |INTEGER ; showAlias diff --git a/x-pack/qa/sql/src/main/resources/command.csv-spec b/x-pack/qa/sql/src/main/resources/command.csv-spec index 879fa486d07f5..06f38f0a07e47 100644 --- a/x-pack/qa/sql/src/main/resources/command.csv-spec +++ b/x-pack/qa/sql/src/main/resources/command.csv-spec @@ -200,89 +200,98 @@ test_alias_emp |ALIAS describeSimpleLike DESCRIBE LIKE 'test_emp'; - column:s | type:s -birth_date | TIMESTAMP -dep | STRUCT -dep.dep_id | VARCHAR -dep.dep_name | VARCHAR -dep.dep_name.keyword | VARCHAR -dep.from_date | TIMESTAMP -dep.to_date | TIMESTAMP -emp_no | INTEGER -first_name | VARCHAR -first_name.keyword | VARCHAR -gender | VARCHAR -hire_date | TIMESTAMP -languages | TINYINT -last_name | VARCHAR -last_name.keyword | VARCHAR -salary | INTEGER + column | type | mapping +--------------------+---------------+--------------- +birth_date |TIMESTAMP |DATE +dep |STRUCT |NESTED +dep.dep_id |VARCHAR |KEYWORD +dep.dep_name |VARCHAR |TEXT +dep.dep_name.keyword|VARCHAR |KEYWORD +dep.from_date |TIMESTAMP |DATE +dep.to_date |TIMESTAMP |DATE +emp_no |INTEGER |INTEGER +extra_gender |VARCHAR |KEYWORD +extra_no |INTEGER |INTEGER +first_name |VARCHAR |TEXT +first_name.keyword |VARCHAR |KEYWORD +gender |VARCHAR |KEYWORD +hire_date |TIMESTAMP |DATE +languages |TINYINT |BYTE +last_name |VARCHAR |TEXT +last_name.keyword |VARCHAR |KEYWORD +salary |INTEGER |INTEGER ; describeMultiLike DESCRIBE LIKE 'test_emp%'; - column:s | type:s -birth_date | TIMESTAMP -dep | STRUCT -dep.dep_id | VARCHAR -dep.dep_name | VARCHAR -dep.dep_name.keyword | VARCHAR -dep.from_date | TIMESTAMP -dep.to_date | TIMESTAMP -emp_no | INTEGER -first_name | VARCHAR -first_name.keyword | VARCHAR -gender | VARCHAR -hire_date | TIMESTAMP -languages | TINYINT -last_name | VARCHAR -last_name.keyword | VARCHAR -salary | INTEGER + column | type | mapping +--------------------+---------------+--------------- +birth_date |TIMESTAMP |DATE +dep |STRUCT |NESTED +dep.dep_id |VARCHAR |KEYWORD +dep.dep_name |VARCHAR |TEXT +dep.dep_name.keyword|VARCHAR |KEYWORD +dep.from_date |TIMESTAMP |DATE +dep.to_date |TIMESTAMP |DATE +emp_no |INTEGER |INTEGER +extra_gender |VARCHAR |KEYWORD +extra_no |INTEGER |INTEGER +first_name |VARCHAR |TEXT +first_name.keyword |VARCHAR |KEYWORD +gender |VARCHAR |KEYWORD +hire_date |TIMESTAMP |DATE +languages |TINYINT |BYTE +last_name |VARCHAR |TEXT +last_name.keyword |VARCHAR |KEYWORD +salary |INTEGER |INTEGER ; describeSimpleIdentifier DESCRIBE "test_emp"; - column:s | type:s -birth_date | TIMESTAMP -dep | STRUCT -dep.dep_id | VARCHAR -dep.dep_name | VARCHAR -dep.dep_name.keyword | VARCHAR -dep.from_date | TIMESTAMP -dep.to_date | TIMESTAMP -emp_no | INTEGER -first_name | VARCHAR -first_name.keyword | VARCHAR -gender | VARCHAR -hire_date | TIMESTAMP -languages | TINYINT -last_name | VARCHAR -last_name.keyword | VARCHAR -salary | INTEGER + column | type | mapping +--------------------+---------------+--------------- +birth_date |TIMESTAMP |DATE +dep |STRUCT |NESTED +dep.dep_id |VARCHAR |KEYWORD +dep.dep_name |VARCHAR |TEXT +dep.dep_name.keyword|VARCHAR |KEYWORD +dep.from_date |TIMESTAMP |DATE +dep.to_date |TIMESTAMP |DATE +emp_no |INTEGER |INTEGER +first_name |VARCHAR |TEXT +first_name.keyword |VARCHAR |KEYWORD +gender |VARCHAR |KEYWORD +hire_date |TIMESTAMP |DATE +languages |TINYINT |BYTE +last_name |VARCHAR |TEXT +last_name.keyword |VARCHAR |KEYWORD +salary |INTEGER |INTEGER ; -describeIncludeExcludeIdentifier -DESCRIBE "test_emp*,-test_emp_*"; - -column:s | type:s -birth_date | TIMESTAMP -dep | STRUCT -dep.dep_id | VARCHAR -dep.dep_name | VARCHAR -dep.dep_name.keyword | VARCHAR -dep.from_date | TIMESTAMP -dep.to_date | TIMESTAMP -emp_no | INTEGER -first_name | VARCHAR -first_name.keyword | VARCHAR -gender | VARCHAR -hire_date | TIMESTAMP -languages | TINYINT -last_name | VARCHAR -last_name.keyword | VARCHAR -salary | INTEGER +// NB: need to pursue how the resolution is done +// should aliases be included or excluded? +describeIncludeExcludeIdentifier-Ignore +DESCRIBE "test_*,-test_alias*"; + + column | type | mapping +--------------------+---------------+--------------- +birth_date |TIMESTAMP |DATE +dep |STRUCT |NESTED +dep.dep_id |VARCHAR |KEYWORD +dep.dep_name |VARCHAR |TEXT +dep.dep_name.keyword|VARCHAR |KEYWORD +dep.from_date |TIMESTAMP |DATE +dep.to_date |TIMESTAMP |DATE +emp_no |INTEGER |INTEGER +first_name |VARCHAR |TEXT +first_name.keyword |VARCHAR |KEYWORD +gender |VARCHAR |KEYWORD +hire_date |TIMESTAMP |DATE +languages |TINYINT |BYTE +last_name |VARCHAR |TEXT +last_name.keyword |VARCHAR |KEYWORD +salary |INTEGER |INTEGER ; - diff --git a/x-pack/qa/sql/src/main/resources/docs.csv-spec b/x-pack/qa/sql/src/main/resources/docs.csv-spec index b0c5936a70299..4d5c8c26b8cd0 100644 --- a/x-pack/qa/sql/src/main/resources/docs.csv-spec +++ b/x-pack/qa/sql/src/main/resources/docs.csv-spec @@ -12,24 +12,24 @@ describeTable // tag::describeTable DESCRIBE emp; - column | type ---------------------+--------------- -birth_date |TIMESTAMP -dep |STRUCT -dep.dep_id |VARCHAR -dep.dep_name |VARCHAR -dep.dep_name.keyword|VARCHAR -dep.from_date |TIMESTAMP -dep.to_date |TIMESTAMP -emp_no |INTEGER -first_name |VARCHAR -first_name.keyword |VARCHAR -gender |VARCHAR -hire_date |TIMESTAMP -languages |TINYINT -last_name |VARCHAR -last_name.keyword |VARCHAR -salary |INTEGER + column | type | mapping +--------------------+---------------+--------------- +birth_date |TIMESTAMP |DATE +dep |STRUCT |NESTED +dep.dep_id |VARCHAR |KEYWORD +dep.dep_name |VARCHAR |TEXT +dep.dep_name.keyword|VARCHAR |KEYWORD +dep.from_date |TIMESTAMP |DATE +dep.to_date |TIMESTAMP |DATE +emp_no |INTEGER |INTEGER +first_name |VARCHAR |TEXT +first_name.keyword |VARCHAR |KEYWORD +gender |VARCHAR |KEYWORD +hire_date |TIMESTAMP |DATE +languages |TINYINT |BYTE +last_name |VARCHAR |TEXT +last_name.keyword |VARCHAR |KEYWORD +salary |INTEGER |INTEGER // end::describeTable ; @@ -51,24 +51,24 @@ showColumns // tag::showColumns SHOW COLUMNS IN emp; - column | type ---------------------+--------------- -birth_date |TIMESTAMP -dep |STRUCT -dep.dep_id |VARCHAR -dep.dep_name |VARCHAR -dep.dep_name.keyword|VARCHAR -dep.from_date |TIMESTAMP -dep.to_date |TIMESTAMP -emp_no |INTEGER -first_name |VARCHAR -first_name.keyword |VARCHAR -gender |VARCHAR -hire_date |TIMESTAMP -languages |TINYINT -last_name |VARCHAR -last_name.keyword |VARCHAR -salary |INTEGER + column | type | mapping +--------------------+---------------+--------------- +birth_date |TIMESTAMP |DATE +dep |STRUCT |NESTED +dep.dep_id |VARCHAR |KEYWORD +dep.dep_name |VARCHAR |TEXT +dep.dep_name.keyword|VARCHAR |KEYWORD +dep.from_date |TIMESTAMP |DATE +dep.to_date |TIMESTAMP |DATE +emp_no |INTEGER |INTEGER +first_name |VARCHAR |TEXT +first_name.keyword |VARCHAR |KEYWORD +gender |VARCHAR |KEYWORD +hire_date |TIMESTAMP |DATE +languages |TINYINT |BYTE +last_name |VARCHAR |TEXT +last_name.keyword |VARCHAR |KEYWORD +salary |INTEGER |INTEGER // end::showColumns ; diff --git a/x-pack/qa/sql/src/main/resources/nested.csv-spec b/x-pack/qa/sql/src/main/resources/nested.csv-spec index 0a188bd7faf13..428ed78120440 100644 --- a/x-pack/qa/sql/src/main/resources/nested.csv-spec +++ b/x-pack/qa/sql/src/main/resources/nested.csv-spec @@ -6,24 +6,24 @@ describeParent DESCRIBE test_emp; -column | type - -birth_date | TIMESTAMP -dep | STRUCT -dep.dep_id | VARCHAR -dep.dep_name | VARCHAR -dep.dep_name.keyword | VARCHAR -dep.from_date | TIMESTAMP -dep.to_date | TIMESTAMP -emp_no | INTEGER -first_name | VARCHAR -first_name.keyword | VARCHAR -gender | VARCHAR -hire_date | TIMESTAMP -languages | TINYINT -last_name | VARCHAR -last_name.keyword | VARCHAR -salary | INTEGER + column | type | mapping +--------------------+---------------+--------------- +birth_date |TIMESTAMP |DATE +dep |STRUCT |NESTED +dep.dep_id |VARCHAR |KEYWORD +dep.dep_name |VARCHAR |TEXT +dep.dep_name.keyword|VARCHAR |KEYWORD +dep.from_date |TIMESTAMP |DATE +dep.to_date |TIMESTAMP |DATE +emp_no |INTEGER |INTEGER +first_name |VARCHAR |TEXT +first_name.keyword |VARCHAR |KEYWORD +gender |VARCHAR |KEYWORD +hire_date |TIMESTAMP |DATE +languages |TINYINT |BYTE +last_name |VARCHAR |TEXT +last_name.keyword |VARCHAR |KEYWORD +salary |INTEGER |INTEGER ; // disable until we figure out how to use field names with . in their name