From 81e9a7c114145c6db9d1c6bef8a5d601b23722c3 Mon Sep 17 00:00:00 2001 From: sjudeng Date: Tue, 10 Oct 2017 20:29:01 -0500 Subject: [PATCH] Support compression key variants between Cassandra 2.x versus 3.x in tests. Increase timeout in log test to resolve test failure. Disregard value order in multi-property hadoop test. Add scripts to improve automation around Dockerized Cassandra. Signed-off-by: sjudeng --- .travis.yml.cassandra | 89 +++++++++++ TESTING.md | 140 ++++++++++++++++-- docs/versions.txt | 2 +- janusgraph-cassandra/config/static/test.crt | 21 +++ .../org/janusgraph/CassandraStorageSetup.java | 10 ++ .../src/test/resources/cqlshrc | 3 + .../src/test/resources/docker-compose.yml | 33 +++++ .../resources/janusgraph_docker_entrypoint.sh | 18 +++ .../src/test/resources/update_config.py | 43 ++++++ .../diskstorage/cql/CQLStoreTest.java | 34 +++-- .../hadoop/AbstractInputFormatIT.java | 10 +- .../hadoop/Cassandra3InputFormatIT.java | 5 +- .../hadoop/CassandraIndexManagementIT.java | 5 +- .../hadoop/CassandraInputFormatIT.java | 5 +- .../janusgraph/hadoop/CassandraScanJobIT.java | 4 +- .../janusgraph/diskstorage/log/LogTest.java | 2 +- 16 files changed, 373 insertions(+), 51 deletions(-) create mode 100644 .travis.yml.cassandra create mode 100644 janusgraph-cassandra/config/static/test.crt create mode 100644 janusgraph-cassandra/src/test/resources/cqlshrc create mode 100644 janusgraph-cassandra/src/test/resources/docker-compose.yml create mode 100755 janusgraph-cassandra/src/test/resources/janusgraph_docker_entrypoint.sh create mode 100755 janusgraph-cassandra/src/test/resources/update_config.py diff --git a/.travis.yml.cassandra b/.travis.yml.cassandra new file mode 100644 index 0000000000..744481ffc4 --- /dev/null +++ b/.travis.yml.cassandra @@ -0,0 +1,89 @@ +# Copyright 2017 JanusGraph Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +dist: trusty +language: java +sudo: true +services: + - docker +jdk: + - openjdk8 + +cache: + directories: + - ${HOME}/.m2 + +env: + matrix: + - MODULE='hadoop-parent/janusgraph-hadoop-2' CASSANDRA_VERSION='3.11.0' ARGS='-DskipHBase -DskipCassandra -DskipCassandra3=false' + - MODULE='cassandra' CASSANDRA_VERSION='3.11.0' CASSANDRA_ENABLE_BOP='true' ARGS='-Dtest=**/thrift/* -Dtest.skip.unordered=true -Dtest.skip.ssl=true -Dtest.skip.serial=true' + - MODULE='cassandra' CASSANDRA_VERSION='3.11.0' ARGS='-Dtest=**/thrift/* -Dtest.skip.ordered=true -Dtest.skip.ssl=true' + - MODULE='cassandra' CASSANDRA_VERSION='3.11.0' CASSANDRA_ENABLE_SSL='true' ARGS='-Dtest=**/thrift/* -Dtest.skip.unordered=true -Dtest.skip.ordered=true -Dtest.skip.serial=true' + - MODULE='cassandra' CASSANDRA_VERSION='3.11.0' CASSANDRA_ENABLE_BOP='true' ARGS='-Dtest=**/astyanax/* -Dtest.skip.unordered=true -Dtest.skip.ssl=true -Dtest.skip.serial=true' + - MODULE='cassandra' CASSANDRA_VERSION='3.11.0' ARGS='-Dtest=**/astyanax/* -Dtest.skip.ordered=true -Dtest.skip.ssl=true' + - MODULE='cassandra' CASSANDRA_VERSION='3.11.0' CASSANDRA_ENABLE_SSL='true' ARGS='-Dtest=**/astyanax/* -Dtest.skip.unordered=true -Dtest.skip.ordered=true -Dtest.skip.serial=true' + - MODULE='cassandra' CASSANDRA_VERSION='3.11.0' ARGS='-Dtest=**/cassandra/*,**/utils/*' + - MODULE='cql' CASSANDRA_VERSION='3.11.0' CASSANDRA_ENABLE_BOP='true' ARGS='-Dtest.skip.murmur=true -Dtest.skip.murmur-ssl=true -Dtest.skip.murmur-serial=true' + - MODULE='cql' CASSANDRA_VERSION='3.11.0' ARGS='-Dtest.skip.byteorderedpartitioner=true -Dtest.skip.murmur-ssl=true' + - MODULE='cql' CASSANDRA_VERSION='3.11.0' CASSANDRA_ENABLE_SSL='true' ARGS='-Dtest.skip.murmur=true -Dtest.skip.byteorderedpartitioner=true -Dtest.skip.murmur-serial=true' + + - MODULE='hadoop-parent/janusgraph-hadoop-2' CASSANDRA_VERSION='3.0.14' ARGS='-DskipHBase -DskipCassandra -DskipCassandra3=false' + - MODULE='cassandra' CASSANDRA_VERSION='3.0.14' CASSANDRA_ENABLE_BOP='true' ARGS='-Dtest=**/thrift/* -Dtest.skip.unordered=true -Dtest.skip.ssl=true -Dtest.skip.serial=true' + - MODULE='cassandra' CASSANDRA_VERSION='3.0.14' ARGS='-Dtest=**/thrift/* -Dtest.skip.ordered=true -Dtest.skip.ssl=true' + - MODULE='cassandra' CASSANDRA_VERSION='3.0.14' CASSANDRA_ENABLE_SSL='true' ARGS='-Dtest=**/thrift/* -Dtest.skip.unordered=true -Dtest.skip.ordered=true -Dtest.skip.serial=true' + - MODULE='cassandra' CASSANDRA_VERSION='3.0.14' CASSANDRA_ENABLE_BOP='true' ARGS='-Dtest=**/astyanax/* -Dtest.skip.unordered=true -Dtest.skip.ssl=true -Dtest.skip.serial=true' + - MODULE='cassandra' CASSANDRA_VERSION='3.0.14' ARGS='-Dtest=**/astyanax/* -Dtest.skip.ordered=true -Dtest.skip.ssl=true' + - MODULE='cassandra' CASSANDRA_VERSION='3.0.14' CASSANDRA_ENABLE_SSL='true' ARGS='-Dtest=**/astyanax/* -Dtest.skip.unordered=true -Dtest.skip.ordered=true -Dtest.skip.serial=true' + - MODULE='cassandra' CASSANDRA_VERSION='3.0.14' ARGS='-Dtest=**/cassandra/*,**/utils/*' + - MODULE='cql' CASSANDRA_VERSION='3.0.14' CASSANDRA_ENABLE_BOP='true' ARGS='-Dtest.skip.murmur=true -Dtest.skip.murmur-ssl=true -Dtest.skip.murmur-serial=true' + - MODULE='cql' CASSANDRA_VERSION='3.0.14' ARGS='-Dtest.skip.byteorderedpartitioner=true -Dtest.skip.murmur-ssl=true' + - MODULE='cql' CASSANDRA_VERSION='3.0.14' CASSANDRA_ENABLE_SSL='true' ARGS='-Dtest.skip.murmur=true -Dtest.skip.byteorderedpartitioner=true -Dtest.skip.murmur-serial=true' + + - MODULE='hadoop-parent/janusgraph-hadoop-2' CASSANDRA_VERSION='2.2.10' ARGS='-DskipHBase' + - MODULE='cassandra' CASSANDRA_VERSION='2.2.10' CASSANDRA_ENABLE_BOP='true' ARGS='-Dtest=**/thrift/* -Dtest.skip.unordered=true -Dtest.skip.ssl=true -Dtest.skip.serial=true' + - MODULE='cassandra' CASSANDRA_VERSION='2.2.10' ARGS='-Dtest=**/thrift/* -Dtest.skip.ordered=true -Dtest.skip.ssl=true' + - MODULE='cassandra' CASSANDRA_VERSION='2.2.10' CASSANDRA_ENABLE_SSL='true' ARGS='-Dtest=**/thrift/* -Dtest.skip.unordered=true -Dtest.skip.ordered=true -Dtest.skip.serial=true' + - MODULE='cassandra' CASSANDRA_VERSION='2.2.10' CASSANDRA_ENABLE_BOP='true' ARGS='-Dtest=**/astyanax/* -Dtest.skip.unordered=true -Dtest.skip.ssl=true -Dtest.skip.serial=true' + - MODULE='cassandra' CASSANDRA_VERSION='2.2.10' ARGS='-Dtest=**/astyanax/* -Dtest.skip.ordered=true -Dtest.skip.ssl=true' + - MODULE='cassandra' CASSANDRA_VERSION='2.2.10' CASSANDRA_ENABLE_SSL='true' ARGS='-Dtest=**/astyanax/* -Dtest.skip.unordered=true -Dtest.skip.ordered=true -Dtest.skip.serial=true' + - MODULE='cassandra' CASSANDRA_VERSION='2.2.10' ARGS='-Dtest=**/cassandra/*,**/utils/*' + - MODULE='cql' CASSANDRA_VERSION='2.2.10' CASSANDRA_ENABLE_BOP='true' ARGS='-Dtest.skip.murmur=true -Dtest.skip.murmur-ssl=true -Dtest.skip.murmur-serial=true' + - MODULE='cql' CASSANDRA_VERSION='2.2.10' ARGS='-Dtest.skip.byteorderedpartitioner=true -Dtest.skip.murmur-ssl=true' + - MODULE='cql' CASSANDRA_VERSION='2.2.10' CASSANDRA_ENABLE_SSL='true' ARGS='-Dtest.skip.murmur=true -Dtest.skip.byteorderedpartitioner=true -Dtest.skip.murmur-serial=true' + +before_install: + - docker-compose -f janusgraph-cassandra/src/test/resources/docker-compose.yml up -d + - until docker exec -it jg-cassandra sh -c 'exec cqlsh -e "show host"' || docker exec -it jg-cassandra sh -c 'exec cqlsh --ssl -e "show host"'; do + >&2 echo "Cassandra is unavailable - sleeping"; + docker-compose -f janusgraph-cassandra/src/test/resources/docker-compose.yml logs; + sleep 1; + done + - echo "Cassandra is up" + - sleep 5 + +install: + - travis_retry travis_wait \ + mvn install --projects janusgraph-${MODULE} --also-make \ + -DskipTests=true -Dmaven.javadoc.skip=true --batch-mode --show-version; + +script: + - travis_retry travis_wait 50 \ + mvn clean verify --projects janusgraph-${MODULE} \ + -Dstorage.hostname=`docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' jg-cassandra` \ + ${ARGS}; + +# Syntax and more info: https://docs.travis-ci.com/user/notifications +notifications: + email: + - janusgraph-ci@googlegroups.com diff --git a/TESTING.md b/TESTING.md index 7299a36b5e..62010c02fe 100644 --- a/TESTING.md +++ b/TESTING.md @@ -56,48 +56,158 @@ mvn test -Dtest=BerkeleyJEGraphPerformanceMemoryTest mvn test -Dtest=BerkeleyJEGraphPerformanceMemoryTest -Dtest.skip.mem=false ``` -### Running Solr tests against a managed Docker container +### Running Tests with an External Solr -Solr tests can be run using a Docker container through the `docker` profile +Solr tests can be run against an external Solr instance. For convenience the `docker` Maven profile is provided to manage a Solr Docker container through the Maven Failsafe Plugin. The default test version will be the same as the Solr client version. ```bash mvn clean install -pl janusgraph-solr -Pdocker ``` -Specify a custom Solr test version using one of the version profiles or the `solr.test.version` property +Additional Maven profiles are defined for testing against default versions of other supported major Solr releases. ```bash mvn clean install -pl janusgraph-solr -Pdocker,solr5 -mvn clean install -pl janusgraph-solr -Pdocker -Dsolr.test.version=5.5.4 ``` -### Running Thrift/CQL tests with ScyllaDB (requires Docker) - -To run Thrift or CQL tests with the [ScyllaDB](http://www.scylladb.com/) backend, ensure Docker is running and run tests under the `scylladb-test` Maven profile. +Finally the `solr.test.version` property can be used to test against arbitrary Solr versions. ```bash -mvn clean install -pl janusgraph-cql -Pscylladb-test +mvn clean install -pl janusgraph-solr -Pdocker -Dsolr.test.version=5.5.4 ``` -### Running Elasticsearch tests against a managed Docker container +### Running Tests with an External Elasticsearch -Elasicsearch tests can be run using a Docker container through the `es-docker` profile +Elasticsearch tests can be run against an external Elasticsearch instance. For convenience the `es-docker` Maven profile is provided to manage an Elasticsearch Docker container through the Maven Failsafe Plugin. The default test version will be the same as the Elasticsearch client version. ```bash mvn clean install -pl janusgraph-es -Pes-docker ``` -Specify a custom Elasticsearch test version using one of the version profiles and optionally the `elasticsearch.docker.test.version` property +Additional Maven profiles are defined for testing against default versions of other supported major Elasticsearch releases. ```bash mvn clean install -pl janusgraph-es -Pes-docker,elasticsearch5 ``` -### Running Hadoop tests with Cassandra-3 using CQL record reader (requires Docker) +Finally the `elasticsearch.docker.test.version` property can be used to test against arbitrary Elasticsearch versions. This is more complicated however because of differences across major versions in required server settings, Docker image names and zipfile artifact availability. The examples below illustrate the differences based on the Elasticsearch major version. + +```bash +mvn clean install -pl janusgraph-es -Pes-docker -Delasticsearch.docker.test.version=5.3.2 +mvn clean install -pl janusgraph-es -Pes-docker -Delasticsearch.test.version=2.3.3 -Delasticsearch.test.groovy.inline="script.engine.groovy.inline.update: true" -Des.docker.image=elasticsearch +mvn clean install -pl janusgraph-es -Pes-docker -Delasticsearch.docker.test.version=1.5.1 -Delasticsearch.test.version=2.4.4 -Delasticsearch.test.groovy.inline="script.disable_dynamic: false" -Des.docker.image=elasticsearch +``` + +### Running Tests with an External Cassandra + +Default and TinkerPop tests can be run against an externally-managed Cassandra instance. For convenience a Docker Compose file is provided in the JanusGraph-Cassandra source distribution for managing a Cassandra instance in a Docker container. + +```bash +CASSANDRA_VERSION=3.11.0 docker-compose -f janusgraph-cassandra/src/test/resources/docker-compose.yml up -d +``` + +Environment variables used when starting the container are described below + +| Variable | Description | Example | +| ---- | ---- | ---- | +| CASSANDRA_VERSION | Docker image version to pull and run | 3.11.0 | +| CASSANDRA_ENABLE_BOP | Enable the `ByteOrderedPartitioner`. Required for TinkerPop tests. | true | +| CASSANDRA_ENABLE_SSL | Enable SSL | true | + +(Optional) Once the instance is started logs can be monitored or alternatively omit the `-d` flag in the above call and run tests in a separate shell + +```bash +docker-compose -f janusgraph-cassandra/src/test/resources/docker-compose.yml logs -f +``` + +Wait for the instance to become available by monitoring logs or programmatically as shown below. Note the two cases being checked are to accommodate both unencrypted and encrypted connections. + +```bash +until docker exec -it jg-cassandra sh -c 'exec cqlsh -e "show host"' || docker exec -it jg-cassandra sh -c 'exec cqlsh --ssl -e "show host"'; do + >&2 echo "Cassandra is unavailable - sleeping"; + sleep 1; +done +``` + +The `storage.hostname` property is used when running tests to indicate an external instance should be used. Depending on the tests being run it may be necessary to provide the Docker container IP address rather than the host address (127.0.0.1) to avoid test failures. The Docker container IP address can be obtained as shown below. + +```bash +STORAGE_HOSTNAME=`docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' jg-cassandra` +``` + +After running tests the container can be stopped and removed as shown below + +```bash +docker-compose -f janusgraph-cassandra/src/test/resources/docker-compose.yml stop +docker-compose -f janusgraph-cassandra/src/test/resources/docker-compose.yml rm -f +``` + +#### Default Tests -To run Hadoop tests with Cassandra-3 using the CQL record reader, start a Cassandra-3 Docker container and run tests with `-DskipCassandra3=false`. Note core HBase and Cassandra-2 Hadoop tests must be skipped when an external Cassandra instance is running. +Default Thrift tests with the `Murmur3Partitioner` partitioner: ```bash -docker run --name jg-cassandra -p 9160:9160 -p 9042:9042 -e CASSANDRA_START_RPC=true -d cassandra:3.10 -mvn clean install -pl :janusgraph-hadoop-2 -DskipHBase -DskipCassandra -DskipCassandra3=false +CASSANDRA_VERSION=3.11.0 docker-compose -f janusgraph-cassandra/src/test/resources/docker-compose.yml up -d +# wait for instance to start (see above) +mvn clean install -pl janusgraph-cassandra -Dtest=**/thrift/* -Dtest.skip.ordered=true -Dtest.skip.ssl=true -Dstorage.hostname=$STORAGE_HOSTNAME +``` + +Default Thrift tests with the `ByteOrderedPartitioner` partitioner: + +```bash +CASSANDRA_VERSION=3.11.0 CASSANDRA_ENABLE_BOP=true docker-compose -f janusgraph-cassandra/src/test/resources/docker-compose.yml up -d +# wait for instance to start (see above) +mvn clean install -pl janusgraph-cassandra -Dtest=**/thrift/* -Dtest.skip.unordered=true -Dtest.skip.ssl=true -Dtest.skip.serial=true -Dstorage.hostname=$STORAGE_HOSTNAME +``` + +Default Thrift SSL tests with the `Murmur3Partitioner` partitioner: + +```bash +CASSANDRA_VERSION=3.11.0 CASSANDRA_ENABLE_SSL=true docker-compose -f janusgraph-cassandra/src/test/resources/docker-compose.yml up -d +# wait for instance to start (see above) +mvn clean install -pl janusgraph-cassandra -Dtest=**/thrift/* -Dtest.skip.unordered=true -Dtest.skip.ordered=true -Dtest.skip.serial=true -Dtest.skip.serial=true -Dstorage.hostname=$STORAGE_HOSTNAME +``` + +To run default Astyanax or CQL tests change the `test` property value in the above calls. Also note that the CQL module uses different property names to toggle the partitioner and enable SSL. + +| Description | Property (Cassandra Module) | Property (CQL Module) | +| ---- | ---- | ---- | +| Skip Murmur3Partitioner tests | test.skip.unordered | test.skip.murmur | +| Skip ByteOrderedPartitioner tests | test.skip.ordered | test.skip.byteorderedpartitioner | +| Skip SSL (murmur) | test.skip.ssl=true | test.skip.murmur-ssl=true | + +#### TinkerPop Tests + +TinkerPop Thrift and CQL tests: + +```bash +CASSANDRA_VERSION=3.11.0 CASSANDRA_ENABLE_BOP=true docker-compose -f janusgraph-cassandra/src/test/resources/docker-compose.yml up -d +# wait for instance to start (see above) +mvn clean install -Dtest.skip.tp=false -DskipTests=true -pl janusgraph-cassandra,janusgraph-cql -fn -Dstorage.hostname=$STORAGE_HOSTNAME +``` + +#### Hadoop Tests + +Hadoop tests with Cassandra 2: + +```bash +CASSANDRA_VERSION=2.2.10 docker-compose -f janusgraph-cassandra/src/test/resources/docker-compose.yml up -d +# wait for instance to start (see above) +mvn clean install -pl :janusgraph-hadoop-2 -DskipHBase -Dstorage.hostname=$STORAGE_HOSTNAME +``` + +Hadoop tests with Cassandra 3 (note that default Cassandra 2 tests must be skipped): + +```bash +CASSANDRA_VERSION=3.11.0 docker-compose -f janusgraph-cassandra/src/test/resources/docker-compose.yml up -d +# wait for instance to start (see above) +mvn clean install -pl :janusgraph-hadoop-2 -DskipHBase -DskipCassandra -DskipCassandra3=false -Dstorage.hostname=$STORAGE_HOSTNAME +``` + +### Running Tests with ScyllaDB + +Thrift and CQL tests can be run against an externally-managed [ScyllaDB](http://www.scylladb.com/) instance. For convenience the `scylladb-test` Maven profile is provided to manage a ScyllaDB Docker container through the Maven Failsafe Plugin. Note this only runs tests with the `Murmur3Partitioner` partitioner and also skips SSL tests. + +```bash +mvn clean install -pl janusgraph-cql -Pscylladb-test ``` diff --git a/docs/versions.txt b/docs/versions.txt index dfae0ba1bb..e9c0a87527 100644 --- a/docs/versions.txt +++ b/docs/versions.txt @@ -22,6 +22,6 @@ software. | JanusGraph | Cassandra | HBase | Bigtable | Elasticsearch | Solr | TinkerPop | Spark | Scala | 0.1.0 | 1.2.z, 2.0.z, 2.1.z | 0.98.z, 1.0.z, 1.1.z, 1.2.z | 0.9.z, 1.0.0-preZ | 1.5.z | 5.2.z | 3.2.z | 1.6.z | 2.10.z | 0.1.1 | 1.2.z, 2.0.z, 2.1.z | 0.98.z, 1.0.z, 1.1.z, 1.2.z | 0.9.z, 1.0.0-preZ | 1.5.z | 5.2.z | 3.2.z | 1.6.z | 2.10.z -| 0.2.0 | 1.2.z, 2.0.z, 2.1.z | 0.98.z, 1.0.z, 1.1.z, 1.2.z, 1.3.z | 0.9.z, 1.0.0-preZ | 1.5-1.7.z, 2.3-2.4.z, 5.z, 6.z | 5.2-5.5.z, 6.2-6.6.z, 7.z | 3.2.z | 1.6.z | 2.10.z +| 0.2.0 | 1.2.z, 2.0.z, 2.1.z, 2.2.z, 3.0.z, 3.11.z | 0.98.z, 1.0.z, 1.1.z, 1.2.z, 1.3.z | 0.9.z, 1.0.0-preZ | 1.5-1.7.z, 2.3-2.4.z, 5.z, 6.z | 5.2-5.5.z, 6.2-6.6.z, 7.z | 3.2.z | 1.6.z | 2.10.z |========================== diff --git a/janusgraph-cassandra/config/static/test.crt b/janusgraph-cassandra/config/static/test.crt new file mode 100644 index 0000000000..607b3d1763 --- /dev/null +++ b/janusgraph-cassandra/config/static/test.crt @@ -0,0 +1,21 @@ +Bag Attributes + friendlyName: cassandra + 2.16.840.1.113894.746875.1.1: +subject=/C=Unknown/ST=Unknown/L=Unknown/O=Unknown/OU=Unknown/CN=localhost +issuer=/C=Unknown/ST=Unknown/L=Unknown/O=Unknown/OU=Unknown/CN=localhost +-----BEGIN CERTIFICATE----- +MIICdjCCAd+gAwIBAgIEctIFMzANBgkqhkiG9w0BAQsFADBuMRAwDgYDVQQGEwdV +bmtub3duMRAwDgYDVQQIEwdVbmtub3duMRAwDgYDVQQHEwdVbmtub3duMRAwDgYD +VQQKEwdVbmtub3duMRAwDgYDVQQLEwdVbmtub3duMRIwEAYDVQQDEwlsb2NhbGhv +c3QwHhcNMTQwOTAxMDczNTI2WhcNMTQxMTMwMDczNTI2WjBuMRAwDgYDVQQGEwdV +bmtub3duMRAwDgYDVQQIEwdVbmtub3duMRAwDgYDVQQHEwdVbmtub3duMRAwDgYD +VQQKEwdVbmtub3duMRAwDgYDVQQLEwdVbmtub3duMRIwEAYDVQQDEwlsb2NhbGhv +c3QwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMcvbCEQJfqxZJoGSLZSrpf1 +1BGnhDLZLorkCcERNKyduiHRWE62M9Y0h7mNcQ4EXaNIeJS9G9LvQrO3ojSyU2bx +vYEgiySnabcZbC7tR4Y9VdW7WRb/qkzmCXRCnj6merQXxsc3UxIRQOBaSnjfDMwv +OaXMG9pJddP1rSYKVLcPAgMBAAGjITAfMB0GA1UdDgQWBBSosZvuR4EuYrpf3mop +uif6CXEUWzANBgkqhkiG9w0BAQsFAAOBgQAAS54akNJUnlfj3a/vCM+Zf0C5wJIy +KMGj+hg1sSs6hrvH4xjti+p8fqMffjazxtjhho2orWd/A0/5pFMzGg1hy+OMu02K +mF9EyveKm5Dw35Y/6o2IXIeAz1/zdQfoPbEniu7MSjyX8E0ZmVWqnombOusOR/tE +mc2ku/RrFXlOUA== +-----END CERTIFICATE----- diff --git a/janusgraph-cassandra/src/test/java/org/janusgraph/CassandraStorageSetup.java b/janusgraph-cassandra/src/test/java/org/janusgraph/CassandraStorageSetup.java index 1bd9d2b1b1..f6993ca4f7 100644 --- a/janusgraph-cassandra/src/test/java/org/janusgraph/CassandraStorageSetup.java +++ b/janusgraph-cassandra/src/test/java/org/janusgraph/CassandraStorageSetup.java @@ -121,6 +121,16 @@ public static WriteConfiguration getCassandraThriftGraphConfiguration(String ks) return getCassandraThriftConfiguration(ks).getConfiguration(); } + public static ModifiableConfiguration getEmbeddedOrThriftConfiguration(String keyspace) { + final ModifiableConfiguration config; + if (HOSTNAME == null) { + config = getEmbeddedConfiguration(keyspace); + } else { + config = getCassandraThriftConfiguration(keyspace); + } + return config; + } + /** * Load cassandra.yaml and data paths from the environment or from default * values if nothing is set in the environment, then delete all existing diff --git a/janusgraph-cassandra/src/test/resources/cqlshrc b/janusgraph-cassandra/src/test/resources/cqlshrc new file mode 100644 index 0000000000..42f9b05b07 --- /dev/null +++ b/janusgraph-cassandra/src/test/resources/cqlshrc @@ -0,0 +1,3 @@ +[ssl] +certfile = /etc/ssl/test.crt +validate = false diff --git a/janusgraph-cassandra/src/test/resources/docker-compose.yml b/janusgraph-cassandra/src/test/resources/docker-compose.yml new file mode 100644 index 0000000000..a73b30110f --- /dev/null +++ b/janusgraph-cassandra/src/test/resources/docker-compose.yml @@ -0,0 +1,33 @@ +# Copyright 2017 JanusGraph Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '2' +services: + cassandra: + image: cassandra:$CASSANDRA_VERSION + container_name: jg-cassandra + ports: + - "9042:9042" + - "9160:9160" + environment: + - CASSANDRA_START_RPC=true + - CASSANDRA_ENABLE_SSL=$CASSANDRA_ENABLE_SSL + - CASSANDRA_ENABLE_BOP=$CASSANDRA_ENABLE_BOP + volumes: + - ./update_config.py:/update_config.py + - ./janusgraph_docker_entrypoint.sh:/janusgraph_docker_entrypoint.sh + - ../../../config/static/test.keystore:/etc/ssl/test.keystore + - ../../../config/static/test.crt:/etc/ssl/test.crt + - ./cqlshrc:/root/.cassandra/cqlshrc + entrypoint: /janusgraph_docker_entrypoint.sh diff --git a/janusgraph-cassandra/src/test/resources/janusgraph_docker_entrypoint.sh b/janusgraph-cassandra/src/test/resources/janusgraph_docker_entrypoint.sh new file mode 100755 index 0000000000..e177dfbe56 --- /dev/null +++ b/janusgraph-cassandra/src/test/resources/janusgraph_docker_entrypoint.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# +# Copyright 2017 JanusGraph Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +/update_config.py +/docker-entrypoint.sh cassandra -f diff --git a/janusgraph-cassandra/src/test/resources/update_config.py b/janusgraph-cassandra/src/test/resources/update_config.py new file mode 100755 index 0000000000..5dfb121daf --- /dev/null +++ b/janusgraph-cassandra/src/test/resources/update_config.py @@ -0,0 +1,43 @@ +#!/usr/bin/python +# +# Copyright 2017 JanusGraph Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os,re + +keystore = '/etc/ssl/test.keystore' +keystore_password = 'cassandra' +enableSsl = os.environ.get('CASSANDRA_ENABLE_SSL','').lower() == 'true' +enableBop = os.environ.get('CASSANDRA_ENABLE_BOP','').lower() == 'true' + +filename = '/etc/cassandra/cassandra.yaml' +with open(filename,'r') as f: + s = f.read() + +if enableSsl: + ssl_conf = re.search('(client_encryption_options:.*?)[\n\r]{2}',s,re.M|re.DOTALL).group(1) + ssl_conf_new = re.sub('enabled:.*','enabled: true', ssl_conf) + ssl_conf_new = re.sub('keystore:.*','keystore: %s' % keystore, ssl_conf_new) + ssl_conf_new = re.sub('keystore_password:.*','keystore_password: %s' % keystore_password, ssl_conf_new) + s = s.replace(ssl_conf, ssl_conf_new) + +if enableBop: + s = re.sub('partitioner:.*', 'partitioner: org.apache.cassandra.dht.ByteOrderedPartitioner', s) + s = re.sub('# initial_token:.*', 'initial_token: 0000000000000000000000000000000000', s) + s = re.sub('num_tokens:','#num_tokens:', s) +else: + s = re.sub('num_tokens:.*','num_tokens: 4', s) + +with open(filename,'w') as f: + f.write(s) diff --git a/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CQLStoreTest.java b/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CQLStoreTest.java index 7d1c1cc358..6b0d750dce 100644 --- a/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CQLStoreTest.java +++ b/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CQLStoreTest.java @@ -19,6 +19,7 @@ import static org.junit.Assert.*; import java.util.Collections; +import java.util.HashMap; import java.util.Map; import org.janusgraph.diskstorage.BackendException; @@ -35,8 +36,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.google.common.collect.ImmutableMap; - public class CQLStoreTest extends KeyColumnValueStoreTest { private static final Logger LOGGER = LoggerFactory.getLogger(CQLStoreTest.class); @@ -102,11 +101,12 @@ public void testDefaultCFCompressor() throws BackendException { final CQLStoreManager cqlStoreManager = openStorageManager(); cqlStoreManager.openDatabase(cf); - final Map defaultCfCompressionOps = new ImmutableMap.Builder() - .put("sstable_compression", DEFAULT_COMPRESSOR_PACKAGE + "." + CF_COMPRESSION_TYPE.getDefaultValue()) - .put("chunk_length_kb", "64") - .build(); - assertEquals(defaultCfCompressionOps, cqlStoreManager.getCompressionOptions(cf)); + final Map opts = cqlStoreManager.getCompressionOptions(cf); + assertEquals(2, opts.size()); + // chunk length key differs between 2.x (chunk_length_kb) and 3.x (chunk_length_in_kb) + assertEquals("64", opts.getOrDefault("chunk_length_kb", opts.get("chunk_length_in_kb"))); + // compression class key differs between 2.x (sstable_compression) and 3.x (class) + assertEquals(DEFAULT_COMPRESSOR_PACKAGE + "." + CF_COMPRESSION_TYPE.getDefaultValue(), opts.getOrDefault("sstable_compression", opts.get("class"))); } @Test @@ -124,13 +124,12 @@ public void testCustomCFCompressor() throws BackendException { // N.B.: clearStorage() truncates CFs but does not delete them mgr.openDatabase(cf); - final Map expected = ImmutableMap - . builder() - .put("sstable_compression", DEFAULT_COMPRESSOR_PACKAGE + "." + cname) - .put("chunk_length_kb", String.valueOf(ckb)) - .build(); - - assertEquals(expected, mgr.getCompressionOptions(cf)); + final Map opts = mgr.getCompressionOptions(cf); + assertEquals(2, opts.size()); + // chunk length key differs between 2.x (chunk_length_kb) and 3.x (chunk_length_in_kb) + assertEquals(String.valueOf(ckb), opts.getOrDefault("chunk_length_kb", opts.get("chunk_length_in_kb"))); + // compression class key differs between 2.x (sstable_compression) and 3.x (class) + assertEquals(DEFAULT_COMPRESSOR_PACKAGE + "." + cname, opts.getOrDefault("sstable_compression", opts.get("class"))); } @Test @@ -144,7 +143,12 @@ public void testDisableCFCompressor() throws BackendException { // N.B.: clearStorage() truncates CFs but does not delete them mgr.openDatabase(cf); - assertEquals(Collections.emptyMap(), mgr.getCompressionOptions(cf)); + final Map opts = new HashMap<>(mgr.getCompressionOptions(cf)); + if ("false".equals(opts.get("enabled"))) { + // Cassandra 3.x contains {"enabled": false"} mapping not found in 2.x + opts.remove("enabled"); + } + assertEquals(Collections.emptyMap(), opts); } @Test diff --git a/janusgraph-hadoop-parent/janusgraph-hadoop-core/src/test/java/org/janusgraph/hadoop/AbstractInputFormatIT.java b/janusgraph-hadoop-parent/janusgraph-hadoop-core/src/test/java/org/janusgraph/hadoop/AbstractInputFormatIT.java index e9c6be4b2a..0fd3ca5012 100644 --- a/janusgraph-hadoop-parent/janusgraph-hadoop-core/src/test/java/org/janusgraph/hadoop/AbstractInputFormatIT.java +++ b/janusgraph-hadoop-parent/janusgraph-hadoop-core/src/test/java/org/janusgraph/hadoop/AbstractInputFormatIT.java @@ -14,6 +14,7 @@ package org.janusgraph.hadoop; +import com.google.common.collect.ImmutableSet; import org.janusgraph.core.Cardinality; import org.janusgraph.core.JanusGraphVertex; import org.janusgraph.example.GraphOfTheGodsFactory; @@ -77,11 +78,10 @@ public void testReadWideVertexWithManyProperties() throws Exception { GraphTraversalSource t = g.traversal(GraphTraversalSource.computer(SparkGraphComputer.class)); assertEquals(numV, (long) t.V().count().next()); propertiesOnVertex = t.V().valueMap().next(); - valuesOnP = (List)propertiesOnVertex.values().iterator().next(); - assertEquals(numProps, valuesOnP.size()); - for (int i = 0; i < numProps; i++) { - assertEquals(Integer.toString(i), valuesOnP.get(i).toString()); - } + final Set observedValuesOnP = ImmutableSet.copyOf((List)propertiesOnVertex.values().iterator().next()); + assertEquals(numProps, observedValuesOnP.size()); + // order may not be preserved in multi-value properties + assertEquals("Unexpected values", ImmutableSet.copyOf(valuesOnP), observedValuesOnP); } @Test diff --git a/janusgraph-hadoop-parent/janusgraph-hadoop-core/src/test/java/org/janusgraph/hadoop/Cassandra3InputFormatIT.java b/janusgraph-hadoop-parent/janusgraph-hadoop-core/src/test/java/org/janusgraph/hadoop/Cassandra3InputFormatIT.java index 133fda5dbe..6b90c610dc 100644 --- a/janusgraph-hadoop-parent/janusgraph-hadoop-core/src/test/java/org/janusgraph/hadoop/Cassandra3InputFormatIT.java +++ b/janusgraph-hadoop-parent/janusgraph-hadoop-core/src/test/java/org/janusgraph/hadoop/Cassandra3InputFormatIT.java @@ -17,13 +17,13 @@ import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.PropertiesConfiguration; import org.janusgraph.CassandraStorageSetup; -import org.janusgraph.diskstorage.configuration.ModifiableConfiguration; import org.janusgraph.diskstorage.configuration.WriteConfiguration; import java.io.IOException; public class Cassandra3InputFormatIT extends CassandraInputFormatIT { + @Override protected PropertiesConfiguration getGraphConfiguration() throws ConfigurationException, IOException { final PropertiesConfiguration config = super.getGraphConfiguration(); config.setProperty("gremlin.hadoop.graphInputFormat", "org.janusgraph.hadoop.formats.cassandra.Cassandra3InputFormat"); @@ -33,7 +33,6 @@ protected PropertiesConfiguration getGraphConfiguration() throws ConfigurationEx @Override public WriteConfiguration getConfiguration() { String className = CassandraInputFormatIT.class.getSimpleName(); - ModifiableConfiguration mc = CassandraStorageSetup.getCassandraThriftConfiguration(className); - return mc.getConfiguration(); + return CassandraStorageSetup.getEmbeddedOrThriftConfiguration(className).getConfiguration(); } } diff --git a/janusgraph-hadoop-parent/janusgraph-hadoop-core/src/test/java/org/janusgraph/hadoop/CassandraIndexManagementIT.java b/janusgraph-hadoop-parent/janusgraph-hadoop-core/src/test/java/org/janusgraph/hadoop/CassandraIndexManagementIT.java index ec00353aca..b671248ec7 100644 --- a/janusgraph-hadoop-parent/janusgraph-hadoop-core/src/test/java/org/janusgraph/hadoop/CassandraIndexManagementIT.java +++ b/janusgraph-hadoop-parent/janusgraph-hadoop-core/src/test/java/org/janusgraph/hadoop/CassandraIndexManagementIT.java @@ -15,15 +15,12 @@ package org.janusgraph.hadoop; import org.janusgraph.CassandraStorageSetup; -import org.janusgraph.diskstorage.configuration.ModifiableConfiguration; import org.janusgraph.diskstorage.configuration.WriteConfiguration; public class CassandraIndexManagementIT extends AbstractIndexManagementIT { @Override public WriteConfiguration getConfiguration() { - String className = getClass().getSimpleName(); - ModifiableConfiguration mc = CassandraStorageSetup.getEmbeddedConfiguration(className); - return mc.getConfiguration(); + return CassandraStorageSetup.getEmbeddedOrThriftConfiguration(getClass().getSimpleName()).getConfiguration(); } } diff --git a/janusgraph-hadoop-parent/janusgraph-hadoop-core/src/test/java/org/janusgraph/hadoop/CassandraInputFormatIT.java b/janusgraph-hadoop-parent/janusgraph-hadoop-core/src/test/java/org/janusgraph/hadoop/CassandraInputFormatIT.java index 5eed49def6..b261278f6a 100644 --- a/janusgraph-hadoop-parent/janusgraph-hadoop-core/src/test/java/org/janusgraph/hadoop/CassandraInputFormatIT.java +++ b/janusgraph-hadoop-parent/janusgraph-hadoop-core/src/test/java/org/janusgraph/hadoop/CassandraInputFormatIT.java @@ -17,7 +17,6 @@ import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.PropertiesConfiguration; import org.janusgraph.CassandraStorageSetup; -import org.janusgraph.diskstorage.configuration.ModifiableConfiguration; import org.janusgraph.diskstorage.configuration.WriteConfiguration; import org.apache.tinkerpop.gremlin.structure.Graph; import org.apache.tinkerpop.gremlin.structure.util.GraphFactory; @@ -44,8 +43,6 @@ protected Graph getGraph() throws ConfigurationException, IOException { @Override public WriteConfiguration getConfiguration() { - String className = getClass().getSimpleName(); - ModifiableConfiguration mc = CassandraStorageSetup.getEmbeddedConfiguration(className); - return mc.getConfiguration(); + return CassandraStorageSetup.getEmbeddedOrThriftConfiguration(getClass().getSimpleName()).getConfiguration(); } } diff --git a/janusgraph-hadoop-parent/janusgraph-hadoop-core/src/test/java/org/janusgraph/hadoop/CassandraScanJobIT.java b/janusgraph-hadoop-parent/janusgraph-hadoop-core/src/test/java/org/janusgraph/hadoop/CassandraScanJobIT.java index 2bfc34eea5..7ec15d88d1 100644 --- a/janusgraph-hadoop-parent/janusgraph-hadoop-core/src/test/java/org/janusgraph/hadoop/CassandraScanJobIT.java +++ b/janusgraph-hadoop-parent/janusgraph-hadoop-core/src/test/java/org/janusgraph/hadoop/CassandraScanJobIT.java @@ -157,9 +157,7 @@ private Job getVertexJobWithDefaultMapper(org.apache.hadoop.conf.Configuration c @Override public WriteConfiguration getConfiguration() { - String className = getClass().getSimpleName(); - ModifiableConfiguration mc = CassandraStorageSetup.getEmbeddedConfiguration(className); - return mc.getConfiguration(); + return CassandraStorageSetup.getEmbeddedOrThriftConfiguration(getClass().getSimpleName()).getConfiguration(); } // public static class NoopScanJob implements ScanJob { diff --git a/janusgraph-test/src/main/java/org/janusgraph/diskstorage/log/LogTest.java b/janusgraph-test/src/main/java/org/janusgraph/diskstorage/log/LogTest.java index f32ffcf88c..69875b0aeb 100644 --- a/janusgraph-test/src/main/java/org/janusgraph/diskstorage/log/LogTest.java +++ b/janusgraph-test/src/main/java/org/janusgraph/diskstorage/log/LogTest.java @@ -207,7 +207,7 @@ public void testFuzzMessagesSerial() throws Exception { StaticBuffer sb = StaticArrayBuffer.of(raw); l.add(sb); expected.add(sb); - Thread.sleep(50L); + Thread.sleep(100L); } reader.await(TIMEOUT_MS); assertEquals(rounds, reader.msgCount);