Update TinkerPop 3.6.1 [tp-tests]

Fixes JanusGraph#3069

* Upgrade hadoop 3.x
* Remove Jackson 1

Signed-off-by: Jan Jansen <jan.jansen@gdata.de>

.github/workflows/ci-index-solr.yml

@@ -65,7 +65,7 @@ jobs:
             args: "-Psolr8"
             name: solr8
             java: 8
-          - module: es
+          - module: solr
             install-args: "-Pjava-11"
             args: "-Psolr8"
             name: solr8

docs/advanced-topics/hadoop.md

@@ -34,7 +34,7 @@ computations of various OLAP queries may be persisted on the Hadoop file
 system.
 
 For configuring a single node Hadoop cluster, please refer to official
-[Apache Hadoop Docs](https://hadoop.apache.org/docs/r{{hadoop2_version }}/hadoop-project-dist/hadoop-common/SingleCluster.html)
+[Apache Hadoop Docs](https://hadoop.apache.org/docs/r{{hadoop_version }}/hadoop-project-dist/hadoop-common/SingleCluster.html)
 
 Once you have a Hadoop cluster up and running, we will need to specify
 the Hadoop configuration files in the `CLASSPATH`. The below document

docs/changelog.md

@@ -27,7 +27,7 @@ All currently supported verions of JanusGraph are listed below.
 | JanusGraph | Storage Version | Cassandra | HBase | Bigtable | Elasticsearch | Solr | TinkerPop | Spark | Scala |
 | ----- | ---- | ---- | ---- | ---- | ---- | ---- | --- | ---- | ---- |
 | 0.6.z | 2 | 3.0.z, 3.11.z | 1.6.z, 2.2.z | 1.3.0, 1.4.0, 1.5.z, 1.6.z, 1.7.z, 1.8.z, 1.9.z, 1.10.z, 1.11.z, 1.14.z | 6.y, 7.y | 7.y, 8.y | 3.5.z | 3.0.z | 2.12.z |
-| 1.0.z | 2 | 3.11.z, 4.0.z | 2.5.z | 1.3.0, 1.4.0, 1.5.z, 1.6.z, 1.7.z, 1.8.z, 1.9.z, 1.10.z, 1.11.z, 1.14.z | 6.y, 7.y | 8.y | 3.5.z | 3.0.z | 2.12.z |
+| 1.0.z | 2 | 3.11.z, 4.0.z | 2.5.z | 1.3.0, 1.4.0, 1.5.z, 1.6.z, 1.7.z, 1.8.z, 1.9.z, 1.10.z, 1.11.z, 1.14.z | 6.y, 7.y | 8.y | 3.6.z | 3.0.z | 2.12.z |
 
 #### End-of-Life
 The versions of JanusGraph listed below are outdated and will no longer receive bugfixes.
@@ -65,7 +65,7 @@ compile "org.janusgraph:janusgraph-core:1.0.0"
 * Elasticsearch 6.0.1, 6.6.0, 7.17.5
 * Apache Lucene 8.11.1
 * Apache Solr 8.11.1
-* Apache TinkerPop 3.5.3
+* Apache TinkerPop 3.6.1
 * Java 8, 11
 
 **Installed versions in the Pre-Packaged Distribution**

janusgraph-all/pom.xml

@@ -104,33 +104,6 @@
             <version>${project.version}</version>
         </dependency>
 
-        <!-- Override Jackson versions in our transitive dependencies.
-             Without these dependencies, janusgraph-all users can end up with
-             1.9 + 1.8 jars together, and this leads to linkage errors.
-             It might be a better idea to dependencyManagement imports if your Maven version supports it:
-             https://maven.apache.org/guides/introduction/introduction-to-dependency-mechanism.html#Importing_Dependencies -->
-        <!-- Jackson 1.x -->
-        <dependency>
-            <groupId>org.codehaus.jackson</groupId>
-            <artifactId>jackson-mapper-asl</artifactId>
-            <version>${jackson1.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.codehaus.jackson</groupId>
-            <artifactId>jackson-core-asl</artifactId>
-            <version>${jackson1.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.codehaus.jackson</groupId>
-            <artifactId>jackson-xc</artifactId>
-            <version>${jackson1.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.codehaus.jackson</groupId>
-            <artifactId>jackson-jaxrs</artifactId>
-            <version>${jackson1.version}</version>
-        </dependency>
-
         <!-- Jackson 2.x -->
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>

janusgraph-core/src/main/java/org/janusgraph/core/JanusGraph.java

@@ -77,6 +77,14 @@
         method = "shouldResetAfterRollback",
         reason = "JanusGraph assumes lifecycle of transactionListeners in AbstractThreadLocalTransaction ends when the " +
             "transaction ends (commit/rollback/close). TinkerPop, however, asserts transactionListeners are active across transactions.")
+@Graph.OptOut(
+        test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MergeVertexTest",
+        method = "g_mergeVXlabel_person_name_markoX_optionXonMatch_age_19X_option",
+        reason = "JanusGraph doesn't support MergeVertex yet.")
+@Graph.OptOut(
+        test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MergeVertexTest",
+        method = "g_withSideEffectXc_label_person_name_markoX_withSideEffectXm_age_19X_mergeVXselectXcXX_optionXonMatch_selectXmXX_option",
+        reason = "JanusGraph doesn't support MergeVertex yet.")
 public interface JanusGraph extends Transaction {
 
     /**

janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/ElementUtils.java

@@ -14,8 +14,6 @@
 
 package org.janusgraph.graphdb.tinkerpop;
 
-import org.apache.tinkerpop.gremlin.structure.Element;
-import org.apache.tinkerpop.gremlin.structure.Graph;
 import org.apache.tinkerpop.gremlin.structure.Vertex;
 import org.janusgraph.core.JanusGraphEdge;
 import org.janusgraph.core.JanusGraphVertex;
@@ -61,13 +59,4 @@ public static RelationIdentifier getEdgeId(Object id) {
         }
         return null;
     }
-
-    public static void verifyArgsMustBeEitherIdOrElement(Object... ids) {
-        assert ids.length>0;
-        int numElements = 0;
-        for (Object id : ids) {
-            if (id instanceof Element) numElements++;
-        }
-        if (numElements>0 && numElements<ids.length) throw Graph.Exceptions.idArgsMustBeEitherIdOrElement();
-    }
 }

janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/JanusGraphBlueprintsTransaction.java

@@ -132,7 +132,6 @@ public JanusGraphVertex addVertex(Object... keyValues) {
     @Override
     public Iterator<Vertex> vertices(Object... vertexIds) {
         if (vertexIds==null || vertexIds.length==0) return (Iterator)getVertices().iterator();
-        ElementUtils.verifyArgsMustBeEitherIdOrElement(vertexIds);
         long[] ids = new long[vertexIds.length];
         int pos = 0;
         for (Object vertexId : vertexIds) {
@@ -147,7 +146,6 @@ public Iterator<Vertex> vertices(Object... vertexIds) {
     @Override
     public Iterator<Edge> edges(Object... edgeIds) {
         if (edgeIds==null || edgeIds.length==0) return (Iterator)getEdges().iterator();
-        ElementUtils.verifyArgsMustBeEitherIdOrElement(edgeIds);
         RelationIdentifier[] ids = new RelationIdentifier[edgeIds.length];
         int pos = 0;
         for (Object edgeId : edgeIds) {

janusgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/optimize/strategy/JanusGraphStepStrategy.java

@@ -22,7 +22,6 @@
 import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
 import org.apache.tinkerpop.gremlin.structure.Element;
 import org.apache.tinkerpop.gremlin.structure.Graph;
-import org.janusgraph.graphdb.tinkerpop.ElementUtils;
 import org.janusgraph.graphdb.tinkerpop.optimize.JanusGraphTraversalUtil;
 import org.janusgraph.graphdb.tinkerpop.optimize.step.HasStepFolder;
 import org.janusgraph.graphdb.tinkerpop.optimize.step.JanusGraphStep;
@@ -63,17 +62,19 @@ public void apply(final Traversal.Admin<?, ?> traversal) {
             } else {
                 //Make sure that any provided "start" elements are instantiated in the current transaction
                 final Object[] ids = originalGraphStep.getIds();
-                ElementUtils.verifyArgsMustBeEitherIdOrElement(ids);
-                if (ids[0] instanceof Element) {
-                    //GraphStep constructor ensures that the entire array is elements
-                    final Object[] elementIds = new Object[ids.length];
-                    for (int i = 0; i < ids.length; i++) {
+                final Object[] elementIds = new Object[ids.length];
+                for (int i = 0; i < ids.length; i++) {
+                    if (ids[i] instanceof Element) {
                         elementIds[i] = ((Element) ids[i]).id();
                     }
-                    originalGraphStep.setIteratorSupplier(() -> originalGraphStep.returnsVertex() ?
-                            ((Graph) originalGraphStep.getTraversal().getGraph().get()).vertices(elementIds) :
-                            ((Graph) originalGraphStep.getTraversal().getGraph().get()).edges(elementIds));
+                    else
+                    {
+                        elementIds[i] = ids[i];
+                    }
                 }
+                originalGraphStep.setIteratorSupplier(() -> originalGraphStep.returnsVertex() ?
+                    ((Graph) originalGraphStep.getTraversal().getGraph().get()).vertices(elementIds) :
+                    ((Graph) originalGraphStep.getTraversal().getGraph().get()).edges(elementIds));
             }
 
         });

janusgraph-core/src/main/java/org/janusgraph/graphdb/util/CloseableIteratorUtils.java

@@ -25,7 +25,7 @@
 
 public class CloseableIteratorUtils {
 
-    public static final CloseableIterator<Object> EMPTY_CLOSABLE_ITERATOR = CloseableIterator.asCloseable(Collections.emptyIterator());
+    public static final CloseableIterator<Object> EMPTY_CLOSABLE_ITERATOR = CloseableIterator.of(Collections.emptyIterator());
 
     public static <T> CloseableIterator<T> emptyIterator() {
         return (CloseableIterator<T>) EMPTY_CLOSABLE_ITERATOR;

janusgraph-cql/src/test/resources/cql-read.properties

@@ -15,6 +15,7 @@
 gremlin.graph=org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph
 gremlin.hadoop.graphReader=org.janusgraph.hadoop.formats.cql.CqlInputFormat
 gremlin.hadoop.graphWriter=org.apache.hadoop.mapreduce.lib.output.NullOutputFormat
+
 gremlin.hadoop.jarsInDistributedCache=true
 gremlin.hadoop.inputLocation=none
 gremlin.hadoop.outputLocation=target/output

janusgraph-hadoop/pom.xml

@@ -77,7 +77,6 @@
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-client</artifactId>
-            <version>${hadoop2.version}</version>
         </dependency>
 
         <!-- JANUSGRAPH -->

janusgraph-hbase/src/test/java/org/janusgraph/hadoop/HBaseInputFormatIT.java

@@ -32,19 +32,24 @@
 @Testcontainers
 public class HBaseInputFormatIT extends AbstractInputFormatIT {
     @Container
-    public static final HBaseContainer hBaseContainer = new HBaseContainer();
+    private static final HBaseContainer hbase = new HBaseContainer();
 
-    protected Graph getGraph() throws IOException, ConfigurationException {
-        final PropertiesConfiguration config = ConfigurationUtil.loadPropertiesConfig("target/test-classes/hbase-read.properties", false);
+    private PropertiesConfiguration getGraphConfiguration(final String filename) throws ConfigurationException, IOException {
+        final PropertiesConfiguration config = ConfigurationUtil.loadPropertiesConfig(filename, false);
         Path baseOutDir = Paths.get((String) config.getProperty("gremlin.hadoop.outputLocation"));
         baseOutDir.toFile().mkdirs();
         String outDir = Files.createTempDirectory(baseOutDir, null).toAbsolutePath().toString();
         config.setProperty("gremlin.hadoop.outputLocation", outDir);
-        return GraphFactory.open(config);
+        return config;
     }
 
     @Override
     public WriteConfiguration getConfiguration() {
-        return hBaseContainer.getWriteConfiguration();
+        return hbase.getWriteConfiguration();
+    }
+
+    @Override
+    protected Graph getGraph() throws ConfigurationException, IOException {
+        return GraphFactory.open(getGraphConfiguration("target/test-classes/hbase-read.properties"));
     }
 }

janusgraph-hbase/src/test/resources/hbase-read-snapshot.properties

@@ -12,22 +12,31 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+#
+# Hadoop Graph Configuration
+#
 gremlin.graph=org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph
 gremlin.hadoop.graphReader=org.janusgraph.hadoop.formats.hbase.HBaseSnapshotInputFormat
 gremlin.hadoop.graphWriter=org.apache.hadoop.mapreduce.lib.output.NullOutputFormat
 
 gremlin.hadoop.jarsInDistributedCache=true
 gremlin.hadoop.inputLocation=none
 gremlin.hadoop.outputLocation=target/output
+
+#
+# JanusGraph HBase InputFormat configuration
+#
 janusgraphmr.ioformat.conf.storage.backend=hbase
 janusgraphmr.ioformat.conf.storage.hostname=localhost
 janusgraphmr.ioformat.conf.storage.hbase.snapshot-name=janusgraph-snapshot
 janusgraphmr.ioformat.conf.storage.hbase.snapshot-restore-dir=target/output
-####################################
-# SparkGraphComputer Configuration #
-####################################
+
+#
+# SparkGraphComputer Configuration
+#
 spark.master=local[4]
 spark.driver.host=127.0.0.1
 spark.executor.memory=1g
 spark.serializer=org.apache.spark.serializer.KryoSerializer
 spark.kryo.registrator=org.janusgraph.hadoop.serialize.JanusGraphKryoRegistrator
+spark.hadoopRDD.ignoreEmptySplits=false

janusgraph-hbase/src/test/resources/hbase-read.properties

@@ -12,20 +12,31 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+#
+# Hadoop Graph Configuration
+#
 gremlin.graph=org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph
 gremlin.hadoop.graphReader=org.janusgraph.hadoop.formats.hbase.HBaseInputFormat
 gremlin.hadoop.graphWriter=org.apache.hadoop.mapreduce.lib.output.NullOutputFormat
 
 gremlin.hadoop.jarsInDistributedCache=true
 gremlin.hadoop.inputLocation=none
 gremlin.hadoop.outputLocation=target/output
+gremlin.spark.persistContext=true
+
+#
+# JanusGraph HBase InputFormat configuration
+#
 janusgraphmr.ioformat.conf.storage.backend=hbase
 janusgraphmr.ioformat.conf.storage.hostname=localhost
-####################################
-# SparkGraphComputer Configuration #
-####################################
+janusgraphmr.ioformat.conf.storage.hbase.table=janusgraph
+
+#
+# SparkGraphComputer Configuration
+#
 spark.master=local[4]
 spark.driver.host=127.0.0.1
 spark.executor.memory=1g
 spark.serializer=org.apache.spark.serializer.KryoSerializer
 spark.kryo.registrator=org.janusgraph.hadoop.serialize.JanusGraphKryoRegistrator
+spark.hadoopRDD.ignoreEmptySplits=false

mkdocs.yml

@@ -168,6 +168,6 @@ nav:
 extra:
   latest_version: 1.0.0
   snapshot_version: 1.0.0-SNAPSHOT
-  tinkerpop_version: 3.5.3
-  hadoop2_version: 2.8.5
+  tinkerpop_version: 3.6.1
+  hadoop_version: 3.3.4
   jamm_version: 0.3.0