diff --git a/docs/configs/janusgraph-cfg.md b/docs/configs/janusgraph-cfg.md
index 44e0dfa3b07..d035a9414d4 100644
--- a/docs/configs/janusgraph-cfg.md
+++ b/docs/configs/janusgraph-cfg.md
@@ -362,7 +362,7 @@ Configuration options to configure batch queries optimization behavior
| Name | Description | Datatype | Default Value | Mutability |
| ---- | ---- | ---- | ---- | ---- |
| query.batch.enabled | Whether traversal queries should be batched when executed against the storage backend. This can lead to significant performance improvement if there is a non-trivial latency to the backend. If `false` then all other configuration options under `query.batch` namespace are ignored. | Boolean | true | MASKABLE |
-| query.batch.has-step-mode | Properties pre-fetching mode for `has` step. Used only when query.batch.enabled is `true`. Supported modes: - `all_properties` Pre-fetch all vertex properties on any property access - `required_properties_only` Pre-fetch necessary vertex properties for the whole chain of foldable `has` steps - `required_and_next_properties` Prefetch the same properties as with `required_properties_only` mode, but also prefetch
+| query.batch.has-step-mode | Properties pre-fetching mode for `has` step. Used only when query.batch.enabled is `true`. Supported modes: - `all_properties` Pre-fetch all vertex properties on any property access (fetches all vertex properties in a single slice query) - `required_properties_only` Pre-fetch necessary vertex properties for the whole chain of foldable `has` steps (uses a separate slice query per each required property) - `required_and_next_properties` Prefetch the same properties as with `required_properties_only` mode, but also prefetch
properties which may be needed in the next properties access step like `values`, `properties,` `valueMap`, `elementMap`, or `propertyMap`.
In case the next step is not one of those properties access steps then this mode behaves same as `required_properties_only`.
In case the next step is one of the properties access steps with limited scope of properties, those properties will be
@@ -372,6 +372,7 @@ behaves same as `all_properties`. - `required_and_next_properties_or_all` Pre
`values`, `properties,` `valueMap`, `elementMap`, or `propertyMap` then acts like `all_properties`. - `none` Skips `has` step batch properties pre-fetch optimization. | String | required_and_next_properties | MASKABLE |
| query.batch.limited | Configure a maximum batch size for queries against the storage backend. This can be used to ensure responsiveness if batches tend to grow very large. The used batch size is equivalent to the barrier size of a preceding `barrier()` step. If a step has no preceding `barrier()`, the default barrier of TinkerPop will be inserted. This option only takes effect if `query.batch.enabled` is `true`. | Boolean | true | MASKABLE |
| query.batch.limited-size | Default batch size (barrier() step size) for queries. This size is applied only for cases where `LazyBarrierStrategy` strategy didn't apply `barrier` step and where user didn't apply barrier step either. This option is used only when `query.batch.limited` is `true`. Notice, value `2147483647` is considered to be unlimited. | Integer | 2500 | MASKABLE |
+| query.batch.properties-mode | Properties pre-fetching mode for `values`, `properties`, `valueMap`, `propertyMap`, `elementMap` steps. Used only when query.batch.enabled is `true`. Supported modes: - `all_properties` Pre-fetch all vertex properties on any property access (fetches all vertex properties in a single slice query) - `required_properties_only` Pre-fetch necessary vertex properties only (uses a separate slice query per each required property) - `none` Skips vertex properties pre-fetching optimization. | String | required_properties_only | MASKABLE |
| query.batch.repeat-step-mode | Batch mode for `repeat` step. Used only when query.batch.enabled is `true`. These modes are controlling how the child steps with batch support are behaving if they placed to the start of the `repeat`, `emit`, or `until` traversals. Supported modes: - `closest_repeat_parent` Child start steps are receiving vertices for batching from the closest `repeat` step parent only. - `all_repeat_parents` Child start steps are receiving vertices for batching from all `repeat` step parents. - `starts_only_of_all_repeat_parents` Child start steps are receiving vertices for batching from the closest `repeat` step parent (both for the parent start and for next iterations) and also from all `repeat` step parents for the parent start. | String | all_repeat_parents | MASKABLE |
### schema
diff --git a/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphTest.java b/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphTest.java
index d763bf8d892..6ecb306888f 100644
--- a/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphTest.java
+++ b/janusgraph-backend-testutils/src/main/java/org/janusgraph/graphdb/JanusGraphTest.java
@@ -130,8 +130,12 @@
import org.janusgraph.graphdb.relations.StandardVertexProperty;
import org.janusgraph.graphdb.serializer.SpecialInt;
import org.janusgraph.graphdb.serializer.SpecialIntSerializer;
+import org.janusgraph.graphdb.tinkerpop.optimize.step.JanusGraphElementMapStep;
import org.janusgraph.graphdb.tinkerpop.optimize.step.JanusGraphHasStep;
+import org.janusgraph.graphdb.tinkerpop.optimize.step.JanusGraphPropertiesStep;
+import org.janusgraph.graphdb.tinkerpop.optimize.step.JanusGraphPropertyMapStep;
import org.janusgraph.graphdb.tinkerpop.optimize.strategy.MultiQueryHasStepStrategyMode;
+import org.janusgraph.graphdb.tinkerpop.optimize.strategy.MultiQueryPropertiesStrategyMode;
import org.janusgraph.graphdb.tinkerpop.optimize.strategy.MultiQueryStrategyRepeatStepMode;
import org.janusgraph.graphdb.transaction.StandardJanusGraphTx;
import org.janusgraph.graphdb.types.CompositeIndexType;
@@ -217,6 +221,7 @@
import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.MAX_COMMIT_TIME;
import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.METRICS_MERGE_STORES;
import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.PARALLEL_BACKEND_OPS;
+import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.PROPERTIES_BATCH_MODE;
import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.PROPERTY_PREFETCHING;
import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.REPEAT_STEP_BATCH_MODE;
import static org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration.REPLACE_INSTANCE_IF_EXISTS;
@@ -4923,6 +4928,7 @@ public void testLimitBatchSizeForMultiQuery() {
TraversalMetrics profile;
testLimitBatchSizeForHasStep(numV, barrierSize, limit, bs, cs);
+ testLimitBatchSizeForPropertySteps(numV, barrierSize, limit, cs);
// test batching for `out()`
profile = testLimitedBatch(() -> graph.traversal().V(bs).barrier(barrierSize).out());
@@ -4933,15 +4939,6 @@ public void testLimitBatchSizeForMultiQuery() {
profile = testLimitedBatch(() -> graph.traversal().V(bs).barrier(barrierSize).out().limit(limit));
assertEquals((int) Math.ceil((double) limit / barrierSize), countBackendQueriesOfSize(barrierSize * 2, profile.getMetrics()));
- // test batching for `values()`
- profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).values("foo"));
- assertEquals(3, countBackendQueriesOfSize(barrierSize, profile.getMetrics()));
- assertEquals(1, countBackendQueriesOfSize(numV - 3 * barrierSize, profile.getMetrics()));
-
- // test early abort with limit for `values()`
- profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).values("foo").limit(limit));
- assertEquals((int) Math.ceil((double) limit / barrierSize), countBackendQueriesOfSize(barrierSize, profile.getMetrics()));
-
// test batching with unlimited batch size
profile = testLimitedBatch(() -> graph.traversal().V(bs).barrier(barrierSize).out(), option(USE_MULTIQUERY), true, option(LIMITED_BATCH), false);
assertEquals(0, countBackendQueriesOfSize(barrierSize, profile.getMetrics()));
@@ -5044,6 +5041,390 @@ public void testLimitBatchSizeForMultiQuery() {
testLimitBatchSizeForMultiQueryOfConnectiveSteps(bs, barrierSize, limit);
}
+ private void testLimitBatchSizeForPropertySteps(int numV, int barrierSize, int limit, JanusGraphVertex[] cs){
+ TraversalMetrics profile;
+
+ // Test required properties prefetching only prefetching
+
+ // test batching for `values()`
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).values("foo"),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), false,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.REQUIRED_PROPERTIES_ONLY.getConfigName());
+ assertEquals(3, countBackendQueriesOfSize(barrierSize, profile.getMetrics()));
+ assertEquals(1, countBackendQueriesOfSize(numV - 3 * barrierSize, profile.getMetrics()));
+
+ // test early abort with limit for `values()`
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).values("foo").limit(limit),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), false,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.REQUIRED_PROPERTIES_ONLY.getConfigName());
+ assertEquals((int) Math.ceil((double) limit / barrierSize), countBackendQueriesOfSize(barrierSize, profile.getMetrics()));
+
+ // test batching for `values()` (enabled prefetching should not influence the query)
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).values("foo"),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), true,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.REQUIRED_PROPERTIES_ONLY.getConfigName());
+ assertEquals(3, countBackendQueriesOfSize(barrierSize, profile.getMetrics()));
+ assertEquals(1, countBackendQueriesOfSize(numV - 3 * barrierSize, profile.getMetrics()));
+
+ // test early abort with limit for `values()` (enabled prefetching should not influence the query)
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).values("foo").limit(limit),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), true,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.REQUIRED_PROPERTIES_ONLY.getConfigName());
+ assertEquals((int) Math.ceil((double) limit / barrierSize), countBackendQueriesOfSize(barrierSize, profile.getMetrics()));
+
+ // test batching for `properties()` (enabled prefetching should not influence the query)
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).properties("foo"),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), true,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.REQUIRED_PROPERTIES_ONLY.getConfigName());
+ assertEquals(3, countBackendQueriesOfSize(barrierSize, profile.getMetrics()));
+ assertEquals(1, countBackendQueriesOfSize(numV - 3 * barrierSize, profile.getMetrics()));
+
+ // test early abort with limit for `values()` (enabled prefetching should not influence the query)
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).properties("foo").limit(limit),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), true,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.REQUIRED_PROPERTIES_ONLY.getConfigName());
+ assertEquals((int) Math.ceil((double) limit / barrierSize), countBackendQueriesOfSize(barrierSize, profile.getMetrics()));
+
+ // test batching for `valueMap()`
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).valueMap("foo"),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), false,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.REQUIRED_PROPERTIES_ONLY.getConfigName());
+ assertEquals(3, countBackendQueriesOfSize(barrierSize, profile.getMetrics()));
+ assertEquals(1, countBackendQueriesOfSize(numV - 3 * barrierSize, profile.getMetrics()));
+
+ // test early abort with limit for `valueMap()`. Limit is going to be placed before valueMap.
+ // Thus, `barrier(barrierSize)` is ignored and instead the default barrier(2500) is used.
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).valueMap("foo").limit(limit),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), false,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.REQUIRED_PROPERTIES_ONLY.getConfigName());
+ assertEquals(1, countBackendQueriesOfSize(limit, profile.getMetrics()));
+
+ // test batching for `valueMap()` (enabled prefetching should not influence the query)
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).valueMap("foo"),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), true,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.REQUIRED_PROPERTIES_ONLY.getConfigName());
+ assertEquals(3, countBackendQueriesOfSize(barrierSize, profile.getMetrics()));
+ assertEquals(1, countBackendQueriesOfSize(numV - 3 * barrierSize, profile.getMetrics()));
+
+ // test early abort with limit for `valueMap()`. Limit is going to be placed before valueMap.
+ // Thus, `barrier(barrierSize)` is ignored and instead the default barrier(2500) is used.
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).valueMap("foo").limit(limit),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), true,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.REQUIRED_PROPERTIES_ONLY.getConfigName());
+ assertEquals(1, countBackendQueriesOfSize(limit, profile.getMetrics()));
+
+ // test batching for `propertyMap()` (enabled prefetching should not influence the query)
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).propertyMap("foo"),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), true,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.REQUIRED_PROPERTIES_ONLY.getConfigName());
+ assertEquals(3, countBackendQueriesOfSize(barrierSize, profile.getMetrics()));
+ assertEquals(1, countBackendQueriesOfSize(numV - 3 * barrierSize, profile.getMetrics()));
+
+ // test early abort with limit for `propertyMap()`. Limit is going to be placed before propertyMap.
+ // Thus, `barrier(barrierSize)` is ignored and instead the default barrier(2500) is used.
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).propertyMap("foo").limit(limit),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), true,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.REQUIRED_PROPERTIES_ONLY.getConfigName());
+ assertEquals(1, countBackendQueriesOfSize(limit, profile.getMetrics()));
+
+ // test batching for `elementMap()`
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).elementMap("foo"),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), false,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.REQUIRED_PROPERTIES_ONLY.getConfigName());
+ assertEquals(6, countBackendQueriesOfSize(barrierSize, profile.getMetrics()));
+ assertEquals(2, countBackendQueriesOfSize(numV - 3 * barrierSize, profile.getMetrics()));
+
+ // test early abort with limit for `elementMap()`. Limit is going to be placed before elementMap.
+ // Thus, `barrier(barrierSize)` is ignored and instead the default barrier(2500) is used.
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).elementMap("foo").limit(limit),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), false,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.REQUIRED_PROPERTIES_ONLY.getConfigName());
+ assertEquals(2, countBackendQueriesOfSize(limit, profile.getMetrics()));
+
+ // test batching for `elementMap()` (enabled prefetching should not influence the query)
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).elementMap("foo"),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), true,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.REQUIRED_PROPERTIES_ONLY.getConfigName());
+ assertEquals(6, countBackendQueriesOfSize(barrierSize, profile.getMetrics()));
+ assertEquals(2, countBackendQueriesOfSize(numV - 3 * barrierSize, profile.getMetrics()));
+
+ // test early abort with limit for `elementMap()`. Limit is going to be placed before elementMap.
+ // Thus, `barrier(barrierSize)` is ignored and instead the default barrier(2500) is used.
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).elementMap("foo").limit(limit),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), true,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.REQUIRED_PROPERTIES_ONLY.getConfigName());
+ assertEquals(2, countBackendQueriesOfSize(limit, profile.getMetrics()));
+
+ // Test all properties prefetching
+
+ int propertiesCount = (int) (graph.traversal().V(cs).values().count().next() / numV);
+
+ // test batching for `values()`
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).values("foo"),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), false,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.ALL_PROPERTIES.getConfigName());
+ assertEquals(3, countBackendQueriesOfSize(barrierSize * propertiesCount, profile.getMetrics()));
+ assertEquals(1, countBackendQueriesOfSize((numV - 3 * barrierSize) * propertiesCount, profile.getMetrics()));
+
+ // test early abort with limit for `values()`
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).values("foo").limit(limit),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), false,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.ALL_PROPERTIES.getConfigName());
+ assertEquals((int) Math.ceil((double) limit / barrierSize), countBackendQueriesOfSize(barrierSize*propertiesCount, profile.getMetrics()));
+
+ // test batching for `values()` (enabled prefetching should not influence the query)
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).values("foo"),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), true,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.ALL_PROPERTIES.getConfigName());
+ assertEquals(3, countBackendQueriesOfSize(barrierSize * propertiesCount, profile.getMetrics()));
+ assertEquals(1, countBackendQueriesOfSize((numV - 3 * barrierSize) * propertiesCount, profile.getMetrics()));
+
+ // test early abort with limit for `values()` (enabled prefetching should not influence the query)
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).values("foo").limit(limit),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), true,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.ALL_PROPERTIES.getConfigName());
+ assertEquals((int) Math.ceil((double) limit / barrierSize), countBackendQueriesOfSize(barrierSize*propertiesCount, profile.getMetrics()));
+
+ // test batching for `properties()` (enabled prefetching should not influence the query)
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).properties("foo"),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), true,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.ALL_PROPERTIES.getConfigName());
+ assertEquals(3, countBackendQueriesOfSize(barrierSize * propertiesCount, profile.getMetrics()));
+ assertEquals(1, countBackendQueriesOfSize((numV - 3 * barrierSize) * propertiesCount, profile.getMetrics()));
+
+ // test early abort with limit for `values()` (enabled prefetching should not influence the query)
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).properties("foo").limit(limit),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), true,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.ALL_PROPERTIES.getConfigName());
+ assertEquals((int) Math.ceil((double) limit / barrierSize), countBackendQueriesOfSize(barrierSize*propertiesCount, profile.getMetrics()));
+
+ // test batching for `valueMap()`
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).valueMap("foo"),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), false,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.ALL_PROPERTIES.getConfigName());
+ assertEquals(3, countBackendQueriesOfSize(barrierSize * propertiesCount, profile.getMetrics()));
+ assertEquals(1, countBackendQueriesOfSize((numV - 3 * barrierSize) * propertiesCount, profile.getMetrics()));
+
+ // test early abort with limit for `valueMap()`. Limit is going to be placed before valueMap.
+ // Thus, `barrier(barrierSize)` is ignored and instead the default barrier(2500) is used.
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).valueMap("foo").limit(limit),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), false,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.ALL_PROPERTIES.getConfigName());
+ assertEquals(1, countBackendQueriesOfSize(limit*propertiesCount, profile.getMetrics()));
+
+ // test batching for `valueMap()` (enabled prefetching should not influence the query)
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).valueMap("foo"),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), true,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.ALL_PROPERTIES.getConfigName());
+ assertEquals(3, countBackendQueriesOfSize(barrierSize * propertiesCount, profile.getMetrics()));
+ assertEquals(1, countBackendQueriesOfSize((numV - 3 * barrierSize) * propertiesCount, profile.getMetrics()));
+
+ // test early abort with limit for `valueMap()`. Limit is going to be placed before valueMap.
+ // Thus, `barrier(barrierSize)` is ignored and instead the default barrier(2500) is used.
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).valueMap("foo").limit(limit),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), true,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.ALL_PROPERTIES.getConfigName());
+ assertEquals(1, countBackendQueriesOfSize(limit*propertiesCount, profile.getMetrics()));
+
+ // test batching for `propertyMap()` (enabled prefetching should not influence the query)
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).propertyMap("foo"),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), true,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.ALL_PROPERTIES.getConfigName());
+ assertEquals(3, countBackendQueriesOfSize(barrierSize * propertiesCount, profile.getMetrics()));
+ assertEquals(1, countBackendQueriesOfSize((numV - 3 * barrierSize) * propertiesCount, profile.getMetrics()));
+
+ // test early abort with limit for `propertyMap()`. Limit is going to be placed before propertyMap.
+ // Thus, `barrier(barrierSize)` is ignored and instead the default barrier(2500) is used.
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).propertyMap("foo").limit(limit),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), true,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.ALL_PROPERTIES.getConfigName());
+ assertEquals(1, countBackendQueriesOfSize(limit*propertiesCount, profile.getMetrics()));
+
+ // test batching for `elementMap()`
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).elementMap("foo"),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), false,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.ALL_PROPERTIES.getConfigName());
+ assertEquals(3, countBackendQueriesOfSize(barrierSize * propertiesCount, profile.getMetrics()));
+ assertEquals(3, countBackendQueriesOfSize(barrierSize, profile.getMetrics()));
+ assertEquals(1, countBackendQueriesOfSize((numV - 3 * barrierSize) * propertiesCount, profile.getMetrics()));
+ assertEquals(1, countBackendQueriesOfSize(numV - 3 * barrierSize, profile.getMetrics()));
+
+ // test early abort with limit for `elementMap()`. Limit is going to be placed before elementMap.
+ // Thus, `barrier(barrierSize)` is ignored and instead the default barrier(2500) is used.
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).elementMap("foo").limit(limit),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), false,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.ALL_PROPERTIES.getConfigName());
+ assertEquals(1, countBackendQueriesOfSize(limit*propertiesCount, profile.getMetrics()));
+ assertEquals(1, countBackendQueriesOfSize(limit, profile.getMetrics()));
+
+ // test batching for `elementMap()` (enabled prefetching should not influence the query)
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).elementMap("foo"),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), true,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.ALL_PROPERTIES.getConfigName());
+ assertEquals(3, countBackendQueriesOfSize(barrierSize * propertiesCount, profile.getMetrics()));
+ assertEquals(3, countBackendQueriesOfSize(barrierSize, profile.getMetrics()));
+ assertEquals(1, countBackendQueriesOfSize((numV - 3 * barrierSize) * propertiesCount, profile.getMetrics()));
+ assertEquals(1, countBackendQueriesOfSize(numV - 3 * barrierSize, profile.getMetrics()));
+
+ // test early abort with limit for `elementMap()`. Limit is going to be placed before elementMap.
+ // Thus, `barrier(barrierSize)` is ignored and instead the default barrier(2500) is used.
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).elementMap("foo").limit(limit),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), true,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.ALL_PROPERTIES.getConfigName());
+ assertEquals(1, countBackendQueriesOfSize(limit*propertiesCount, profile.getMetrics()));
+ assertEquals(1, countBackendQueriesOfSize(limit, profile.getMetrics()));
+
+ // Test disabled properties preFetching mode
+
+ // test no batching used for `values()`
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).values("foo"),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), false,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.NONE.getConfigName());
+ assertEquals(100, countBackendQueriesOfSize(s -> true, profile.getMetrics()));
+ assertNull(getLastStepMetrics(profile, JanusGraphPropertiesStep.class).getAnnotation("multi"));
+
+ // test no batching used for `values()` (enabled prefetching should not influence the query)
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).values("foo"),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), true,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.NONE.getConfigName());
+ assertEquals(100, countOptimizationQueries(profile.getMetrics()));
+ assertNull(getLastStepMetrics(profile, JanusGraphPropertiesStep.class).getAnnotation("multi"));
+
+ // test no batching used for `properties()` (enabled prefetching should not influence the query)
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).properties("foo"),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), false,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.NONE.getConfigName());
+ assertEquals(100, countBackendQueriesOfSize(s -> true, profile.getMetrics()));
+ assertNull(getLastStepMetrics(profile, JanusGraphPropertiesStep.class).getAnnotation("multi"));
+
+ // test no batching used for `properties()` (enabled prefetching should not influence the query)
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).properties("foo"),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), true,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.NONE.getConfigName());
+ assertEquals(100, countOptimizationQueries(profile.getMetrics()));
+ assertNull(getLastStepMetrics(profile, JanusGraphPropertiesStep.class).getAnnotation("multi"));
+
+ // test no batching used for `valueMap()`
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).valueMap("foo"),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), false,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.NONE.getConfigName());
+ assertNull(getLastStepMetrics(profile, JanusGraphPropertyMapStep.class).getAnnotation("multi"));
+
+ // test no batching used for `valueMap()` (enabled prefetching should not influence the query)
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).valueMap("foo"),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), true,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.NONE.getConfigName());
+ assertNull(getLastStepMetrics(profile, JanusGraphPropertyMapStep.class).getAnnotation("multi"));
+
+ // test no batching used for `propertyMap()` (enabled prefetching should not influence the query)
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).propertyMap("foo"),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), false,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.NONE.getConfigName());
+ assertNull(getLastStepMetrics(profile, JanusGraphPropertyMapStep.class).getAnnotation("multi"));
+
+ // test no batching used for `propertyMap()` (enabled prefetching should not influence the query)
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).propertyMap("foo"),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), true,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.NONE.getConfigName());
+ assertNull(getLastStepMetrics(profile, JanusGraphPropertyMapStep.class).getAnnotation("multi"));
+
+ // test no batching used for `elementMap()`
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).elementMap("foo"),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), false,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.NONE.getConfigName());
+ assertNull(getLastStepMetrics(profile, JanusGraphElementMapStep.class).getAnnotation("multi"));
+
+ // test no batching used for `elementMap()` (enabled prefetching should not influence the query)
+ profile = testLimitedBatch(() -> graph.traversal().V(cs).barrier(barrierSize).elementMap("foo"),
+ option(USE_MULTIQUERY), true,
+ option(LIMITED_BATCH), true,
+ option(PROPERTY_PREFETCHING), true,
+ option(PROPERTIES_BATCH_MODE), MultiQueryPropertiesStrategyMode.NONE.getConfigName());
+ assertNull(getLastStepMetrics(profile, JanusGraphElementMapStep.class).getAnnotation("multi"));
+ }
+
private void testLimitBatchSizeForHasStep(int numV, int barrierSize, int limit, JanusGraphVertex[] bs, JanusGraphVertex[] cs){
TraversalMetrics profile;
diff --git a/janusgraph-core/src/main/java/org/janusgraph/core/TransactionBuilder.java b/janusgraph-core/src/main/java/org/janusgraph/core/TransactionBuilder.java
index da7eb0e4083..dba638eb913 100644
--- a/janusgraph-core/src/main/java/org/janusgraph/core/TransactionBuilder.java
+++ b/janusgraph-core/src/main/java/org/janusgraph/core/TransactionBuilder.java
@@ -16,6 +16,7 @@
import org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration;
import org.janusgraph.graphdb.tinkerpop.optimize.strategy.MultiQueryHasStepStrategyMode;
+import org.janusgraph.graphdb.tinkerpop.optimize.strategy.MultiQueryPropertiesStrategyMode;
import java.time.Instant;
@@ -68,7 +69,7 @@ public interface TransactionBuilder {
TransactionBuilder propertyPrefetching(boolean enabled);
/**
- * Enable or disable multi-query, i.e. query.batch
+ * Enable or disable multi-query, i.e. `query.batch.enabled`
*
* @param enabled
* @return Object containing properties that will enable/disable multi-query
@@ -151,12 +152,21 @@ public interface TransactionBuilder {
/**
* Sets `has` step strategy mode.
*
- * Doesn't have any effect if multi-query was disabled via config `query.batch`.
+ * Doesn't have any effect if multi-query was disabled via config `query.batch.enabled = false`.
*
* @return Object with the set `has` step strategy mode settings
*/
TransactionBuilder setHasStepStrategyMode(MultiQueryHasStepStrategyMode hasStepStrategyMode);
+ /**
+ * Sets properties strategy mode.
+ *
+ * Doesn't have any effect if multi-query was disabled via config `query.batch.enabled = false`.
+ *
+ * @return Object with the set properties strategy mode settings
+ */
+ TransactionBuilder setPropertiesStrategyMode(MultiQueryPropertiesStrategyMode propertiesStrategyMode);
+
/**
* Sets the group name for this transaction which provides a way for gathering
* reporting on multiple transactions into one group.
diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/configuration/GraphDatabaseConfiguration.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/configuration/GraphDatabaseConfiguration.java
index b10f6092810..15a0aab0310 100644
--- a/janusgraph-core/src/main/java/org/janusgraph/graphdb/configuration/GraphDatabaseConfiguration.java
+++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/configuration/GraphDatabaseConfiguration.java
@@ -58,6 +58,7 @@
import org.janusgraph.graphdb.query.index.BruteForceIndexSelectionStrategy;
import org.janusgraph.graphdb.query.index.IndexSelectionStrategy;
import org.janusgraph.graphdb.query.index.ThresholdBasedIndexSelectionStrategy;
+import org.janusgraph.graphdb.tinkerpop.optimize.strategy.MultiQueryPropertiesStrategyMode;
import org.janusgraph.graphdb.tinkerpop.optimize.strategy.MultiQueryStrategyRepeatStepMode;
import org.janusgraph.graphdb.tinkerpop.optimize.strategy.MultiQueryHasStepStrategyMode;
import org.janusgraph.graphdb.transaction.StandardTransactionBuilder;
@@ -331,8 +332,8 @@ public class GraphDatabaseConfiguration {
public static final ConfigOption HAS_STEP_BATCH_MODE = new ConfigOption<>(QUERY_BATCH_NS,"has-step-mode",
String.format("Properties pre-fetching mode for `has` step. Used only when "+USE_MULTIQUERY.toStringWithoutRoot()+" is `true`. " +
"Supported modes: " +
- "- `%s` Pre-fetch all vertex properties on any property access " +
- "- `%s` Pre-fetch necessary vertex properties for the whole chain of foldable `has` steps " +
+ "- `%s` Pre-fetch all vertex properties on any property access (fetches all vertex properties in a single slice query) " +
+ "- `%s` Pre-fetch necessary vertex properties for the whole chain of foldable `has` steps (uses a separate slice query per each required property) " +
"- `%s` Prefetch the same properties as with `%s` mode, but also prefetch\n" +
"properties which may be needed in the next properties access step like `values`, `properties,` `valueMap`, `elementMap`, or `propertyMap`.\n" +
"In case the next step is not one of those properties access steps then this mode behaves same as `%s`.\n" +
@@ -355,6 +356,17 @@ public class GraphDatabaseConfiguration {
MultiQueryHasStepStrategyMode.NONE.getConfigName()),
ConfigOption.Type.MASKABLE, MultiQueryHasStepStrategyMode.REQUIRED_AND_NEXT_PROPERTIES.getConfigName());
+ public static final ConfigOption PROPERTIES_BATCH_MODE = new ConfigOption<>(QUERY_BATCH_NS,"properties-mode",
+ String.format("Properties pre-fetching mode for `values`, `properties`, `valueMap`, `propertyMap`, `elementMap` steps. Used only when "+USE_MULTIQUERY.toStringWithoutRoot()+" is `true`. " +
+ "Supported modes: " +
+ "- `%s` Pre-fetch all vertex properties on any property access (fetches all vertex properties in a single slice query) " +
+ "- `%s` Pre-fetch necessary vertex properties only (uses a separate slice query per each required property) " +
+ "- `%s` Skips vertex properties pre-fetching optimization. ",
+ MultiQueryPropertiesStrategyMode.ALL_PROPERTIES.getConfigName(),
+ MultiQueryPropertiesStrategyMode.REQUIRED_PROPERTIES_ONLY.getConfigName(),
+ MultiQueryPropertiesStrategyMode.NONE.getConfigName()),
+ ConfigOption.Type.MASKABLE, MultiQueryPropertiesStrategyMode.REQUIRED_PROPERTIES_ONLY.getConfigName());
+
// ################ SCHEMA #######################
// ################################################
@@ -1326,6 +1338,7 @@ public boolean apply(@Nullable String s) {
private String metricsPrefix;
private String unknownIndexKeyName;
private MultiQueryHasStepStrategyMode hasStepStrategyMode;
+ private MultiQueryPropertiesStrategyMode propertiesStrategyMode;
private StoreFeatures storeFeatures = null;
@@ -1444,6 +1457,10 @@ public MultiQueryHasStepStrategyMode hasStepStrategyMode() {
return hasStepStrategyMode;
}
+ public MultiQueryPropertiesStrategyMode propertiesStrategyMode() {
+ return propertiesStrategyMode;
+ }
+
public boolean adjustQueryLimit() {
return adjustQueryLimit;
}
@@ -1572,6 +1589,7 @@ private void preLoadConfiguration() {
limitedBatchSize = configuration.get(LIMITED_BATCH_SIZE);
repeatStepMode = selectExactConfig(REPEAT_STEP_BATCH_MODE, MultiQueryStrategyRepeatStepMode.values());
hasStepStrategyMode = selectExactConfig(HAS_STEP_BATCH_MODE, MultiQueryHasStepStrategyMode.values());
+ propertiesStrategyMode = selectExactConfig(PROPERTIES_BATCH_MODE, MultiQueryPropertiesStrategyMode.values());
indexSelectionStrategy = Backend.getImplementationClass(configuration, configuration.get(INDEX_SELECT_STRATEGY),
REGISTERED_INDEX_SELECTION_STRATEGIES);
diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/database/StandardJanusGraph.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/database/StandardJanusGraph.java
index 9a52f2c2cd8..0bd3e58b90e 100644
--- a/janusgraph-core/src/main/java/org/janusgraph/graphdb/database/StandardJanusGraph.java
+++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/database/StandardJanusGraph.java
@@ -65,8 +65,9 @@
import org.janusgraph.graphdb.database.cache.SchemaCache;
import org.janusgraph.graphdb.database.idassigner.VertexIDAssigner;
import org.janusgraph.graphdb.database.idhandling.IDHandler;
-import org.janusgraph.graphdb.tinkerpop.optimize.strategy.JanusGraphUnusedMultiQueryRemovalStrategy;
import org.janusgraph.graphdb.tinkerpop.optimize.strategy.JanusGraphHasStepStrategy;
+import org.janusgraph.graphdb.tinkerpop.optimize.strategy.JanusGraphLocalQueryOptimizerStrategy;
+import org.janusgraph.graphdb.tinkerpop.optimize.strategy.JanusGraphUnusedMultiQueryRemovalStrategy;
import org.janusgraph.util.IDUtils;
import org.janusgraph.graphdb.database.index.IndexInfoRetriever;
import org.janusgraph.graphdb.database.index.IndexUpdate;
@@ -90,7 +91,6 @@
import org.janusgraph.graphdb.tinkerpop.optimize.strategy.AdjacentVertexHasUniquePropertyOptimizerStrategy;
import org.janusgraph.graphdb.tinkerpop.optimize.strategy.AdjacentVertexIsOptimizerStrategy;
import org.janusgraph.graphdb.tinkerpop.optimize.strategy.JanusGraphIoRegistrationStrategy;
-import org.janusgraph.graphdb.tinkerpop.optimize.strategy.JanusGraphLocalQueryOptimizerStrategy;
import org.janusgraph.graphdb.tinkerpop.optimize.strategy.JanusGraphMixedIndexAggStrategy;
import org.janusgraph.graphdb.tinkerpop.optimize.strategy.JanusGraphMixedIndexCountStrategy;
import org.janusgraph.graphdb.tinkerpop.optimize.strategy.JanusGraphMultiQueryStrategy;
@@ -147,12 +147,12 @@ public class StandardJanusGraph extends JanusGraphBlueprintsGraph {
AdjacentVertexIsOptimizerStrategy.instance(),
AdjacentVertexHasUniquePropertyOptimizerStrategy.instance(),
JanusGraphLocalQueryOptimizerStrategy.instance(),
+ JanusGraphHasStepStrategy.instance(),
JanusGraphMultiQueryStrategy.instance(),
JanusGraphUnusedMultiQueryRemovalStrategy.instance(),
JanusGraphMixedIndexAggStrategy.instance(),
JanusGraphMixedIndexCountStrategy.instance(),
JanusGraphStepStrategy.instance(),
- JanusGraphHasStepStrategy.instance(),
JanusGraphIoRegistrationStrategy.instance());
//Register with cache
diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/step/JanusGraphElementMapStep.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/step/JanusGraphElementMapStep.java
new file mode 100644
index 00000000000..ead23d5e8c3
--- /dev/null
+++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/step/JanusGraphElementMapStep.java
@@ -0,0 +1,186 @@
+// Copyright 2023 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.graphdb.tinkerpop.optimize.step;
+
+import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
+import org.apache.tinkerpop.gremlin.process.traversal.step.Profiling;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.ElementMapStep;
+import org.apache.tinkerpop.gremlin.process.traversal.util.MutableMetrics;
+import org.apache.tinkerpop.gremlin.structure.Element;
+import org.apache.tinkerpop.gremlin.structure.Property;
+import org.apache.tinkerpop.gremlin.structure.T;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.janusgraph.core.BaseVertexQuery;
+import org.janusgraph.core.JanusGraphProperty;
+import org.janusgraph.graphdb.query.profile.QueryProfiler;
+import org.janusgraph.graphdb.query.vertex.BasicVertexCentricQueryBuilder;
+import org.janusgraph.graphdb.query.vertex.BasicVertexCentricQueryUtil;
+import org.janusgraph.graphdb.tinkerpop.optimize.step.fetcher.LabelStepBatchFetcher;
+import org.janusgraph.graphdb.tinkerpop.optimize.step.fetcher.PropertiesStepBatchFetcher;
+import org.janusgraph.graphdb.tinkerpop.profile.TP3ProfileWrapper;
+import org.janusgraph.graphdb.util.CopyStepUtil;
+import org.janusgraph.graphdb.util.JanusGraphTraverserUtil;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Set;
+
+public class JanusGraphElementMapStep extends ElementMapStep implements Profiling, PropertiesPrefetchingStep> {
+
+ private boolean useMultiQuery = false;
+ private LabelStepBatchFetcher labelStepBatchFetcher;
+ private PropertiesStepBatchFetcher propertiesStepBatchFetcher;
+ private QueryProfiler queryProfiler = QueryProfiler.NO_OP;
+ private int batchSize = Integer.MAX_VALUE;
+ private boolean prefetchAllPropertiesRequired;
+ private boolean prefetchingAllowed = true;
+ private final Set propertyKeysSet;
+
+ public JanusGraphElementMapStep(ElementMapStep originalStep) {
+ super(originalStep.getTraversal(), originalStep.getPropertyKeys());
+ CopyStepUtil.copyAbstractStepModifiableFields(originalStep, this);
+ if(originalStep.isOnGraphComputer()){
+ onGraphComputer();
+ }
+ propertyKeysSet = new HashSet<>(Arrays.asList(getPropertyKeys()));
+ if (originalStep instanceof JanusGraphElementMapStep) {
+ JanusGraphElementMapStep originalJanusGraphElementMapStep = (JanusGraphElementMapStep) originalStep;
+ prefetchingAllowed = originalJanusGraphElementMapStep.prefetchingAllowed;;
+ setBatchSize(originalJanusGraphElementMapStep.batchSize);
+ setUseMultiQuery(originalJanusGraphElementMapStep.useMultiQuery);
+ }
+ }
+
+ @Override
+ protected Map map(final Traverser.Admin traverser) {
+ if (useMultiQuery && traverser.get() instanceof Vertex) {
+ Vertex vertexToFetch = (Vertex) traverser.get();
+ int loops = JanusGraphTraverserUtil.getLoops(traverser);
+ Map map = new LinkedHashMap();
+ addElementProperties(map, vertexToFetch, loops);
+ addIncludedOptions(map, vertexToFetch, loops);
+ return (Map) map;
+ }
+ return super.map(traverser);
+ }
+
+ private void addElementProperties(Map map, Vertex vertexToFetch, int loops){
+ Iterator properties = propertiesStepBatchFetcher
+ .fetchData(getTraversal(), vertexToFetch, loops).iterator();
+ while (properties.hasNext()) {
+ final Property property = properties.next();
+ String propertyKey = property.key();
+ if(!prefetchAllPropertiesRequired || propertyKeysSet.isEmpty() || propertyKeysSet.contains(propertyKey)){
+ map.put(propertyKey, property.value());
+ }
+ }
+ }
+
+ private void addIncludedOptions(Map map, Vertex vertexToFetch, int loops){
+ map.put(T.id, vertexToFetch.id());
+ map.put(T.label, labelStepBatchFetcher.fetchData(getTraversal(), vertexToFetch, loops));
+ }
+
+ @Override
+ public void setMetrics(MutableMetrics metrics) {
+ queryProfiler = new TP3ProfileWrapper(metrics);
+ }
+
+ @Override
+ public void setUseMultiQuery(boolean useMultiQuery) {
+ this.useMultiQuery = prefetchingAllowed && useMultiQuery;
+ if(useMultiQuery){
+ if(propertiesStepBatchFetcher == null){
+ propertiesStepBatchFetcher = new PropertiesStepBatchFetcher(JanusGraphElementMapStep.this::makePropertiesQuery, batchSize);
+ }
+ if(labelStepBatchFetcher == null){
+ labelStepBatchFetcher = new LabelStepBatchFetcher(JanusGraphElementMapStep.this::makeLabelsQuery, batchSize);
+ }
+ }
+ }
+
+ private Q makeLabelsQuery(Q query) {
+ return (Q) BasicVertexCentricQueryUtil.withLabelVertices((BasicVertexCentricQueryBuilder) query)
+ .profiler(queryProfiler);
+ }
+
+
+ private Q makePropertiesQuery(Q query) {
+ if(!prefetchAllPropertiesRequired){
+ query.keys(getPropertyKeys());
+ }
+ ((BasicVertexCentricQueryBuilder) query).profiler(queryProfiler);
+ return query;
+ }
+
+
+ @Override
+ public void setBatchSize(int batchSize) {
+ this.batchSize = batchSize;
+ if(labelStepBatchFetcher != null){
+ labelStepBatchFetcher.setBatchSize(batchSize);
+ }
+ if(propertiesStepBatchFetcher != null){
+ propertiesStepBatchFetcher.setBatchSize(batchSize);
+ }
+ }
+
+ @Override
+ public void registerFirstNewLoopFutureVertexForPrefetching(Vertex futureVertex, int futureVertexTraverserLoop) {
+ if(useMultiQuery){
+ labelStepBatchFetcher.registerFirstNewLoopFutureVertexForPrefetching(futureVertex);
+ propertiesStepBatchFetcher.registerFirstNewLoopFutureVertexForPrefetching(futureVertex);
+ }
+ }
+
+ @Override
+ public void registerSameLoopFutureVertexForPrefetching(Vertex futureVertex, int futureVertexTraverserLoop) {
+ if(useMultiQuery){
+ labelStepBatchFetcher.registerCurrentLoopFutureVertexForPrefetching(futureVertex, futureVertexTraverserLoop);
+ propertiesStepBatchFetcher.registerCurrentLoopFutureVertexForPrefetching(futureVertex, futureVertexTraverserLoop);
+ }
+ }
+
+ @Override
+ public void registerNextLoopFutureVertexForPrefetching(Vertex futureVertex, int futureVertexTraverserLoop) {
+ if(useMultiQuery){
+ labelStepBatchFetcher.registerNextLoopFutureVertexForPrefetching(futureVertex, futureVertexTraverserLoop);
+ propertiesStepBatchFetcher.registerNextLoopFutureVertexForPrefetching(futureVertex, futureVertexTraverserLoop);
+ }
+ }
+
+ @Override
+ public boolean isPrefetchAllPropertiesRequired() {
+ return prefetchAllPropertiesRequired;
+ }
+
+ @Override
+ public void setPrefetchAllPropertiesRequired(boolean prefetchAllPropertiesRequired) {
+ this.prefetchAllPropertiesRequired = prefetchAllPropertiesRequired;
+ }
+
+ @Override
+ public boolean isPrefetchingAllowed() {
+ return prefetchingAllowed;
+ }
+
+ @Override
+ public void setPrefetchingAllowed(boolean prefetchingAllowed) {
+ this.prefetchingAllowed = prefetchingAllowed;
+ }
+}
diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/step/JanusGraphPropertiesStep.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/step/JanusGraphPropertiesStep.java
index c0e31976385..37934ef0470 100644
--- a/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/step/JanusGraphPropertiesStep.java
+++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/step/JanusGraphPropertiesStep.java
@@ -40,18 +40,21 @@
import org.janusgraph.graphdb.tinkerpop.optimize.JanusGraphTraversalUtil;
import org.janusgraph.graphdb.tinkerpop.optimize.step.fetcher.PropertiesStepBatchFetcher;
import org.janusgraph.graphdb.tinkerpop.profile.TP3ProfileWrapper;
-import org.janusgraph.graphdb.util.JanusGraphTraverserUtil;
import org.janusgraph.graphdb.util.CopyStepUtil;
+import org.janusgraph.graphdb.util.JanusGraphTraverserUtil;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
+import java.util.Set;
/**
* @author Matthias Broecheler (me@matthiasb.com)
*/
-public class JanusGraphPropertiesStep extends PropertiesStep implements HasStepFolder, Profiling, MultiQueriable {
+public class JanusGraphPropertiesStep extends PropertiesStep implements HasStepFolder, Profiling, PropertiesPrefetchingStep {
private boolean useMultiQuery = false;
private QueryProfiler queryProfiler = QueryProfiler.NO_OP;
@@ -59,13 +62,18 @@ public class JanusGraphPropertiesStep extends PropertiesStep implements Ha
private PropertiesStepBatchFetcher propertiesStepBatchFetcher;
private int batchSize = Integer.MAX_VALUE;
+ private boolean prefetchAllPropertiesRequired;
+ private final Set propertyKeysSet;
+ private boolean prefetchingAllowed = true;
public JanusGraphPropertiesStep(PropertiesStep originalStep) {
super(originalStep.getTraversal(), originalStep.getReturnType(), originalStep.getPropertyKeys());
CopyStepUtil.copyAbstractStepModifiableFields(originalStep, this);
+ propertyKeysSet = new HashSet<>(Arrays.asList(getPropertyKeys()));
if (originalStep instanceof JanusGraphPropertiesStep) {
JanusGraphPropertiesStep originalJanusGraphPropertiesStep = (JanusGraphPropertiesStep) originalStep;
+ this.prefetchingAllowed = originalJanusGraphPropertiesStep.prefetchingAllowed;
setUseMultiQuery(originalJanusGraphPropertiesStep.useMultiQuery);
this.hasContainers = originalJanusGraphPropertiesStep.hasContainers;
this.limit = originalJanusGraphPropertiesStep.limit;
@@ -77,7 +85,7 @@ public JanusGraphPropertiesStep(PropertiesStep originalStep) {
@Override
public void setUseMultiQuery(boolean useMultiQuery) {
- this.useMultiQuery = useMultiQuery;
+ this.useMultiQuery = prefetchingAllowed && useMultiQuery;
if(useMultiQuery && propertiesStepBatchFetcher == null){
propertiesStepBatchFetcher = new PropertiesStepBatchFetcher(JanusGraphPropertiesStep.this::makeQuery, batchSize);
}
@@ -105,8 +113,13 @@ public void registerNextLoopFutureVertexForPrefetching(Vertex futureVertex, int
}
private Q makeQuery(Q query) {
- final String[] keys = getPropertyKeys();
- query.keys(keys);
+ return makeQuery(query, prefetchAllPropertiesRequired);
+ }
+
+ private Q makeQuery(Q query, boolean prefetchAllPropertiesRequired) {
+ if(!prefetchAllPropertiesRequired){
+ query.keys(getPropertyKeys());
+ }
for (final HasContainer condition : hasContainers) {
query.has(condition.getKey(), JanusGraphPredicateUtils.convert(condition.getBiPredicate()), condition.getValue());
}
@@ -117,11 +130,19 @@ private Q makeQuery(Q query) {
}
private Iterator convertIterator(Iterable iterable) {
+ return convertIterator(iterable, prefetchAllPropertiesRequired);
+ }
+
+ private Iterator convertIterator(Iterable iterable, boolean prefetchAllPropertiesRequired) {
+ Iterator propertiesIt = iterable.iterator();
+ if(prefetchAllPropertiesRequired && !propertyKeysSet.isEmpty()){
+ propertiesIt = Iterators.filter(propertiesIt, property -> propertyKeysSet.contains(property.key()));
+ }
if (getReturnType().forProperties()) {
- return (Iterator) iterable.iterator();
+ return (Iterator) propertiesIt;
}
assert getReturnType().forValues();
- return (Iterator) Iterators.transform(iterable.iterator(), Property::value);
+ return (Iterator) Iterators.transform(propertiesIt, Property::value);
}
/**
@@ -136,8 +157,8 @@ protected Iterator flatMap(final Traverser.Admin traverser) {
if (useMultiQuery && elementToFetchDataFor instanceof Vertex) {
return convertIterator(propertiesStepBatchFetcher.fetchData(getTraversal(), (Vertex) elementToFetchDataFor, JanusGraphTraverserUtil.getLoops(traverser)));
} else if (elementToFetchDataFor instanceof JanusGraphVertex || elementToFetchDataFor instanceof WrappedVertex) {
- final JanusGraphVertexQuery query = makeQuery((JanusGraphTraversalUtil.getJanusGraphVertex(traverser)).query());
- return convertIterator(query.properties());
+ final JanusGraphVertexQuery query = makeQuery((JanusGraphTraversalUtil.getJanusGraphVertex(traverser)).query(), false);
+ return convertIterator(query.properties(), false);
} else {
//It is some other element (edge or vertex property)
Iterator iterator;
@@ -252,4 +273,23 @@ public void setMetrics(MutableMetrics metrics) {
queryProfiler = new TP3ProfileWrapper(metrics);
}
+ @Override
+ public boolean isPrefetchAllPropertiesRequired() {
+ return prefetchAllPropertiesRequired;
+ }
+
+ @Override
+ public void setPrefetchAllPropertiesRequired(boolean prefetchAllPropertiesRequired) {
+ this.prefetchAllPropertiesRequired = prefetchAllPropertiesRequired;
+ }
+
+ @Override
+ public boolean isPrefetchingAllowed() {
+ return prefetchingAllowed;
+ }
+
+ @Override
+ public void setPrefetchingAllowed(boolean prefetchingAllowed) {
+ this.prefetchingAllowed = prefetchingAllowed;
+ }
}
diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/step/JanusGraphPropertyMapStep.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/step/JanusGraphPropertyMapStep.java
new file mode 100644
index 00000000000..6abc4afa91a
--- /dev/null
+++ b/janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/step/JanusGraphPropertyMapStep.java
@@ -0,0 +1,225 @@
+// Copyright 2023 JanusGraph Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package org.janusgraph.graphdb.tinkerpop.optimize.step;
+
+import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
+import org.apache.tinkerpop.gremlin.process.traversal.step.Profiling;
+import org.apache.tinkerpop.gremlin.process.traversal.step.map.PropertyMapStep;
+import org.apache.tinkerpop.gremlin.process.traversal.step.util.WithOptions;
+import org.apache.tinkerpop.gremlin.process.traversal.util.MutableMetrics;
+import org.apache.tinkerpop.gremlin.structure.Element;
+import org.apache.tinkerpop.gremlin.structure.Property;
+import org.apache.tinkerpop.gremlin.structure.PropertyType;
+import org.apache.tinkerpop.gremlin.structure.T;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.janusgraph.core.BaseVertexQuery;
+import org.janusgraph.core.JanusGraphProperty;
+import org.janusgraph.graphdb.query.profile.QueryProfiler;
+import org.janusgraph.graphdb.query.vertex.BasicVertexCentricQueryBuilder;
+import org.janusgraph.graphdb.query.vertex.BasicVertexCentricQueryUtil;
+import org.janusgraph.graphdb.tinkerpop.optimize.step.fetcher.LabelStepBatchFetcher;
+import org.janusgraph.graphdb.tinkerpop.optimize.step.fetcher.PropertiesStepBatchFetcher;
+import org.janusgraph.graphdb.tinkerpop.profile.TP3ProfileWrapper;
+import org.janusgraph.graphdb.util.CopyStepUtil;
+import org.janusgraph.graphdb.util.JanusGraphTraverserUtil;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class JanusGraphPropertyMapStep extends PropertyMapStep implements Profiling, PropertiesPrefetchingStep> {
+
+ private boolean useMultiQuery = false;
+ private LabelStepBatchFetcher labelStepBatchFetcher;
+ private PropertiesStepBatchFetcher propertiesStepBatchFetcher;
+ private QueryProfiler queryProfiler = QueryProfiler.NO_OP;
+ private int batchSize = Integer.MAX_VALUE;
+ private boolean withIdsFetching;
+ private boolean withLabelsFetching;
+ private boolean prefetchAllPropertiesRequired;
+ private final Set propertyKeysSet;
+ private boolean prefetchingAllowed = true;
+
+ public JanusGraphPropertyMapStep(PropertyMapStep originalStep) {
+ super(originalStep.getTraversal(), originalStep.getReturnType(), originalStep.getPropertyKeys());
+ CopyStepUtil.copyAbstractStepModifiableFields(originalStep, this);
+ tokens = originalStep.getIncludedTokens();
+ withIdsFetching = includeToken(WithOptions.ids);
+ withLabelsFetching = includeToken(WithOptions.labels);
+ traversalRing = originalStep.getTraversalRing();
+ traversalRing.getTraversals().forEach(this::integrateChild);
+ parameters = originalStep.getParameters();
+ parameters.getTraversals().forEach(this::integrateChild);
+ propertyKeysSet = new HashSet<>(Arrays.asList(getPropertyKeys()));
+
+ if (originalStep instanceof JanusGraphPropertyMapStep) {
+ JanusGraphPropertyMapStep originalJanusGraphPropertyMapStep = (JanusGraphPropertyMapStep) originalStep;
+ prefetchingAllowed = originalJanusGraphPropertyMapStep.prefetchingAllowed;
+ setBatchSize(originalJanusGraphPropertyMapStep.batchSize);
+ setUseMultiQuery(originalJanusGraphPropertyMapStep.useMultiQuery);
+ }
+ }
+
+ @Override
+ public void configure(final Object... keyValues) {
+ super.configure(keyValues);
+ withIdsFetching = includeToken(WithOptions.ids);
+ withLabelsFetching = includeToken(WithOptions.labels);
+ createLabelFetcherIfNeeded();
+ }
+
+ @Override
+ protected Map map(final Traverser.Admin traverser) {
+ if (useMultiQuery && traverser.get() instanceof Vertex) {
+ Vertex vertexToFetch = (Vertex) traverser.get();
+ int loops = JanusGraphTraverserUtil.getLoops(traverser);
+ Map map = new LinkedHashMap();
+ addElementProperties(map, vertexToFetch, loops);
+ addIncludedOptions(map, vertexToFetch, loops);
+ applyTraversalRingToMap(map);
+ return (Map) map;
+ }
+ return super.map(traverser);
+ }
+
+ private void addElementProperties(Map map, Vertex vertexToFetch, int loops){
+ Iterator properties = propertiesStepBatchFetcher
+ .fetchData(getTraversal(), vertexToFetch, loops).iterator();
+ while(properties.hasNext()) {
+ Property property = (Property)properties.next();
+ String propertyKey = property.key();
+ if(!prefetchAllPropertiesRequired || propertyKeysSet.isEmpty() || propertyKeysSet.contains(propertyKey)){
+ Object value = this.returnType == PropertyType.VALUE ? property.value() : property;
+ map.compute(propertyKey, (k, v) -> {
+ final List