From f34200b9ad849909293622a4f43172c4b8d7c699 Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Wed, 5 Jun 2013 16:11:59 -0400 Subject: [PATCH 01/50] Inital copy of titan-hbase to titan-accumulo. --- pom.xml | 1 + titan-accumulo/bin/get-active-master.rb | 45 ++ titan-accumulo/bin/graceful_stop.sh | 104 +++++ titan-accumulo/bin/hbase | 341 +++++++++++++++ titan-accumulo/bin/hbase-config.sh | 118 ++++++ titan-accumulo/bin/hbase-daemon.sh | 202 +++++++++ titan-accumulo/bin/hbase-daemons.sh | 55 +++ titan-accumulo/bin/local-master-backup.sh | 55 +++ titan-accumulo/bin/local-regionservers.sh | 54 +++ titan-accumulo/bin/master-backup.sh | 76 ++++ titan-accumulo/bin/regionservers.sh | 75 ++++ titan-accumulo/bin/rolling-restart.sh | 162 +++++++ titan-accumulo/bin/start-hbase.sh | 54 +++ titan-accumulo/bin/stop-hbase.sh | 73 ++++ titan-accumulo/bin/zookeepers.sh | 61 +++ .../config/hadoop-metrics.properties | 70 +++ titan-accumulo/config/hbase-env.sh | 94 +++++ titan-accumulo/config/hbase-policy.xml | 53 +++ titan-accumulo/config/hbase-site.xml | 37 ++ titan-accumulo/config/log4j.properties | 74 ++++ titan-accumulo/config/regionservers | 1 + titan-accumulo/config/whirr-hbase.properties | 36 ++ titan-accumulo/pom.xml | 154 +++++++ titan-accumulo/src/assembly/distribution.xml | 100 +++++ .../hbase/HBaseKeyColumnValueStore.java | 307 ++++++++++++++ .../diskstorage/hbase/HBaseStoreManager.java | 399 ++++++++++++++++++ .../diskstorage/hbase/HBaseTransaction.java | 19 + .../src/main/resources/log4j.properties | 13 + .../src/main/resources/titan.properties | 2 + .../src/test/config/hadoop-metrics.properties | 70 +++ titan-accumulo/src/test/config/hbase-env.sh | 94 +++++ .../src/test/config/hbase-policy.xml | 53 +++ titan-accumulo/src/test/config/hbase-site.xml | 38 ++ .../src/test/config/log4j.properties | 74 ++++ titan-accumulo/src/test/config/regionservers | 1 + .../src/test/config/whirr-hbase.properties | 36 ++ .../titan/HBaseStorageSetup.java | 119 ++++++ .../titan/blueprints/HBaseBlueprintsTest.java | 53 +++ .../hbase/HBaseKeyColumnValueTest.java | 28 ++ .../HBaseLockKeyColumnValueStoreTest.java | 22 + ...BaseMultiWriteKeyColumnValueStoreTest.java | 26 ++ .../hbase/HBaseGraphConcurrentTest.java | 18 + .../hbase/HBaseGraphPerformanceTest.java | 18 + .../titan/graphdb/hbase/HBaseGraphTest.java | 18 + .../src/test/resources/log4j.properties | 14 + .../src/test/resources/rexster-fragment.xml | 13 + 46 files changed, 3530 insertions(+) create mode 100755 titan-accumulo/bin/get-active-master.rb create mode 100755 titan-accumulo/bin/graceful_stop.sh create mode 100755 titan-accumulo/bin/hbase create mode 100755 titan-accumulo/bin/hbase-config.sh create mode 100755 titan-accumulo/bin/hbase-daemon.sh create mode 100755 titan-accumulo/bin/hbase-daemons.sh create mode 100755 titan-accumulo/bin/local-master-backup.sh create mode 100755 titan-accumulo/bin/local-regionservers.sh create mode 100755 titan-accumulo/bin/master-backup.sh create mode 100755 titan-accumulo/bin/regionservers.sh create mode 100755 titan-accumulo/bin/rolling-restart.sh create mode 100755 titan-accumulo/bin/start-hbase.sh create mode 100755 titan-accumulo/bin/stop-hbase.sh create mode 100755 titan-accumulo/bin/zookeepers.sh create mode 100644 titan-accumulo/config/hadoop-metrics.properties create mode 100644 titan-accumulo/config/hbase-env.sh create mode 100644 titan-accumulo/config/hbase-policy.xml create mode 100644 titan-accumulo/config/hbase-site.xml create mode 100644 titan-accumulo/config/log4j.properties create mode 100644 titan-accumulo/config/regionservers create mode 100644 titan-accumulo/config/whirr-hbase.properties create mode 100644 titan-accumulo/pom.xml create mode 100644 titan-accumulo/src/assembly/distribution.xml create mode 100644 titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseKeyColumnValueStore.java create mode 100644 titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseStoreManager.java create mode 100644 titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseTransaction.java create mode 100644 titan-accumulo/src/main/resources/log4j.properties create mode 100644 titan-accumulo/src/main/resources/titan.properties create mode 100644 titan-accumulo/src/test/config/hadoop-metrics.properties create mode 100644 titan-accumulo/src/test/config/hbase-env.sh create mode 100644 titan-accumulo/src/test/config/hbase-policy.xml create mode 100644 titan-accumulo/src/test/config/hbase-site.xml create mode 100644 titan-accumulo/src/test/config/log4j.properties create mode 100644 titan-accumulo/src/test/config/regionservers create mode 100644 titan-accumulo/src/test/config/whirr-hbase.properties create mode 100644 titan-accumulo/src/test/java/com/thinkaurelius/titan/HBaseStorageSetup.java create mode 100644 titan-accumulo/src/test/java/com/thinkaurelius/titan/blueprints/HBaseBlueprintsTest.java create mode 100644 titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseKeyColumnValueTest.java create mode 100644 titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseLockKeyColumnValueStoreTest.java create mode 100644 titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseMultiWriteKeyColumnValueStoreTest.java create mode 100644 titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/hbase/HBaseGraphConcurrentTest.java create mode 100644 titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/hbase/HBaseGraphPerformanceTest.java create mode 100644 titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/hbase/HBaseGraphTest.java create mode 100644 titan-accumulo/src/test/resources/log4j.properties create mode 100644 titan-accumulo/src/test/resources/rexster-fragment.xml diff --git a/pom.xml b/pom.xml index 0913523a5e..dc3f603428 100644 --- a/pom.xml +++ b/pom.xml @@ -78,6 +78,7 @@ titan-lucene titan-hbase titan-all + titan-accumulo ${basedir}/target diff --git a/titan-accumulo/bin/get-active-master.rb b/titan-accumulo/bin/get-active-master.rb new file mode 100755 index 0000000000..8887a4574c --- /dev/null +++ b/titan-accumulo/bin/get-active-master.rb @@ -0,0 +1,45 @@ +#!/usr/bin/env hbase-jruby +# Copyright 2011 The Apache Software Foundation +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with this +# work for additional information regarding copyright ownership. The ASF +# licenses this file to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Prints the hostname of the machine running the active master. + +include Java +import org.apache.hadoop.hbase.HBaseConfiguration +import org.apache.hadoop.hbase.ServerName +import org.apache.hadoop.hbase.zookeeper.ZKUtil +import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher + +# disable debug/info logging on this script for clarity +log_level = org.apache.log4j.Level::ERROR +org.apache.log4j.Logger.getLogger('org.apache.hadoop.hbase').setLevel(log_level) +org.apache.log4j.Logger.getLogger('org.apache.zookeeper').setLevel(log_level) + +config = HBaseConfiguration.create + +zk = ZooKeeperWatcher.new(config, 'get-active-master', nil) +begin + master_address = ZKUtil.getData(zk, zk.masterAddressZNode) + if master_address + puts ServerName.parseVersionedServerName(master_address).getHostname() + else + puts 'Master not running' + end +ensure + zk.close() +end + diff --git a/titan-accumulo/bin/graceful_stop.sh b/titan-accumulo/bin/graceful_stop.sh new file mode 100755 index 0000000000..beac4b15b1 --- /dev/null +++ b/titan-accumulo/bin/graceful_stop.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +# +#/** +# * Copyright 2011 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# Move regions off a server then stop it. Optionally restart and reload. +# Turn off the balancer before running this script. +function usage { + echo "Usage: graceful_stop.sh [--config ] [--restart [--reload]] [--thrift] [--rest] " + echo " thrift If we should stop/start thrift before/after the hbase stop/start" + echo " rest If we should stop/start rest before/after the hbase stop/start" + echo " restart If we should restart after graceful stop" + echo " reload Move offloaded regions back on to the restarted server" + echo " debug Print helpful debug information" + echo " hostname Hostname of server we are to stop" + exit 1 +} + +if [ $# -lt 1 ]; then + usage +fi + +bin=`dirname "$0"` +bin=`cd "$bin">/dev/null; pwd` +# This will set HBASE_HOME, etc. +. "$bin"/hbase-config.sh +# Get arguments +restart= +reload= +debug= +thrift= +rest= +while [ $# -gt 0 ] +do + case "$1" in + --thrift) thrift=true; shift;; + --rest) rest=true; shift;; + --restart) restart=true; shift;; + --reload) reload=true; shift;; + --debug) debug="--debug"; shift;; + --) shift; break;; + -*) usage ;; + *) break;; # terminate while loop + esac +done + +# "$@" contains the rest. Must be at least the hostname left. +if [ $# -lt 1 ]; then + usage +fi + +hostname=$1 +filename="/tmp/$hostname" +# Run the region mover script. +echo "Disabling balancer!" +echo 'balance_switch false' | "$bin"/hbase --config ${HBASE_CONF_DIR} shell +echo "Unloading $hostname region(s)" +HBASE_NOEXEC=true "$bin"/hbase --config ${HBASE_CONF_DIR} org.jruby.Main "$bin"/region_mover.rb --file=$filename $debug unload $hostname +echo "Unloaded $hostname region(s)" +# Stop the server. Have to put hostname into its own little file for hbase-daemons.sh +hosts="/tmp/$(basename $0).$$.tmp" +echo $hostname >> $hosts +if [ "$thrift" != "" ]; then + "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} stop thrift +fi +if [ "$rest" != "" ]; then + "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} stop rest +fi +"$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} stop regionserver +if [ "$restart" != "" ]; then + "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} start regionserver + if [ "$thrift" != "" ]; then + # -b 0.0.0.0 says listen on all interfaces rather than just default. + "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} start thrift -b 0.0.0.0 + fi + if [ "$rest" != "" ]; then + "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} start rest + fi + if [ "$reload" != "" ]; then + echo "Reloading $hostname region(s)" + HBASE_NOEXEC=true "$bin"/hbase --config ${HBASE_CONF_DIR} org.jruby.Main "$bin"/region_mover.rb --file=$filename $debug load $hostname + echo "Reloaded $hostname region(s)" + fi +fi + +# Cleanup tmp files. +trap "rm -f "/tmp/$(basename $0).*.tmp" &> /dev/null" EXIT diff --git a/titan-accumulo/bin/hbase b/titan-accumulo/bin/hbase new file mode 100755 index 0000000000..62702057be --- /dev/null +++ b/titan-accumulo/bin/hbase @@ -0,0 +1,341 @@ +#! /usr/bin/env bash +# +#/** +# * Copyright 2007 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# +# The hbase command script. Based on the hadoop command script putting +# in hbase classes, libs and configurations ahead of hadoop's. +# +# TODO: Narrow the amount of duplicated code. +# +# Environment Variables: +# +# JAVA_HOME The java implementation to use. Overrides JAVA_HOME. +# +# HBASE_CLASSPATH Extra Java CLASSPATH entries. +# +# HBASE_HEAPSIZE The maximum amount of heap to use, in MB. +# Default is 1000. +# +# HBASE_LIBRARY_PATH HBase additions to JAVA_LIBRARY_PATH for adding +# native libaries. +# +# HBASE_OPTS Extra Java runtime options. +# +# HBASE_CONF_DIR Alternate conf dir. Default is ${HBASE_HOME}/conf. +# +# HBASE_ROOT_LOGGER The root appender. Default is INFO,console +# +# MAVEN_HOME Where mvn is installed. +# +# JRUBY_HOME JRuby path: $JRUBY_HOME/lib/jruby.jar should exist. +# Defaults to the jar packaged with HBase. +# +# JRUBY_OPTS Extra options (eg '--1.9') passed to the hbase shell. +# Empty by default. +# +bin=`dirname "$0"` +bin=`cd "$bin">/dev/null; pwd` + +# This will set HBASE_HOME, etc. +. "$bin"/hbase-config.sh + +cygwin=false +case "`uname`" in +CYGWIN*) cygwin=true;; +esac + +# Detect if we are in hbase sources dir +in_dev_env=false +if [ -d "${HBASE_HOME}/target" ]; then + in_dev_env=true +fi + +# if no args specified, show usage +if [ $# = 0 ]; then + echo "Usage: hbase " + echo "where an option from one of these categories:" + echo "" + echo "DBA TOOLS" + echo " shell run the HBase shell" + echo " hbck run the hbase 'fsck' tool" + echo " hlog write-ahead-log analyzer" + echo " hfile store file analyzer" + echo " zkcli run the ZooKeeper shell" + echo "" + echo "PROCESS MANAGEMENT" + echo " master run an HBase HMaster node" + echo " regionserver run an HBase HRegionServer node" + echo " zookeeper run a Zookeeper server" + echo " rest run an HBase REST server" + echo " thrift run the HBase Thrift server" + echo " thrift2 run the HBase Thrift2 server" + echo " avro run an HBase Avro server" + echo "" + echo "PACKAGE MANAGEMENT" + echo " classpath dump hbase CLASSPATH" + echo " version print the version" + echo "" + echo " or" + echo " CLASSNAME run the class named CLASSNAME" + echo "Most commands print help when invoked w/o parameters." + exit 1 +fi + +# get arguments +COMMAND=$1 +shift + +JAVA=$JAVA_HOME/bin/java +JAVA_HEAP_MAX=-Xmx1000m + +MVN="mvn" +if [ "$MAVEN_HOME" != "" ]; then + MVN=${MAVEN_HOME}/bin/mvn +fi + +# override default settings for this command, if applicable +if [ -f "$HBASE_HOME/conf/hbase-env-$COMMAND.sh" ]; then + . "$HBASE_HOME/conf/hbase-env-$COMMAND.sh" +fi + +# check envvars which might override default args +if [ "$HBASE_HEAPSIZE" != "" ]; then + #echo "run with heapsize $HBASE_HEAPSIZE" + JAVA_HEAP_MAX="-Xmx""$HBASE_HEAPSIZE""m" + #echo $JAVA_HEAP_MAX +fi + +# so that filenames w/ spaces are handled correctly in loops below +IFS= + +# CLASSPATH initially contains $HBASE_CONF_DIR +CLASSPATH="${HBASE_CONF_DIR}" +CLASSPATH=${CLASSPATH}:$JAVA_HOME/lib/tools.jar + +add_maven_deps_to_classpath() { + # Need to generate classpath from maven pom. This is costly so generate it + # and cache it. Save the file into our target dir so a mvn clean will get + # clean it up and force us create a new one. + f="${HBASE_HOME}/target/cached_classpath.txt" + if [ ! -f "${f}" ] + then + ${MVN} -f "${HBASE_HOME}/pom.xml" dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null + fi + CLASSPATH=${CLASSPATH}:`cat "${f}"` +} + +add_maven_main_classes_to_classpath() { + if [ -d "$HBASE_HOME/target/classes" ]; then + CLASSPATH=${CLASSPATH}:$HBASE_HOME/target/classes + fi +} + +add_maven_test_classes_to_classpath() { + # For developers, add hbase classes to CLASSPATH + f="$HBASE_HOME/target/test-classes" + if [ -d "${f}" ]; then + CLASSPATH=${CLASSPATH}:${f} + fi +} + +# Add maven target directory +if $in_dev_env; then + add_maven_deps_to_classpath + add_maven_main_classes_to_classpath + add_maven_test_classes_to_classpath +fi + +# For releases, add hbase & webapps to CLASSPATH +# Webapps must come first else it messes up Jetty +if [ -d "$HBASE_HOME/hbase-webapps" ]; then + CLASSPATH=${CLASSPATH}:$HBASE_HOME +fi +if [ -d "$HBASE_HOME/target/hbase-webapps" ]; then + CLASSPATH="${CLASSPATH}:${HBASE_HOME}/target" +fi +for f in $HBASE_HOME/hbase*.jar; do + if [[ $f = *sources.jar ]] + then + : # Skip sources.jar + elif [ -f $f ] + then + CLASSPATH=${CLASSPATH}:$f; + fi +done + +# Add libs to CLASSPATH +for f in $HBASE_HOME/lib/*.jar; do + CLASSPATH=${CLASSPATH}:$f; +done + +# Add user-specified CLASSPATH last +if [ "$HBASE_CLASSPATH" != "" ]; then + CLASSPATH=${CLASSPATH}:${HBASE_CLASSPATH} +fi + +# default log directory & file +if [ "$HBASE_LOG_DIR" = "" ]; then + HBASE_LOG_DIR="$HBASE_HOME/logs" +fi +if [ "$HBASE_LOGFILE" = "" ]; then + HBASE_LOGFILE='hbase.log' +fi + +# cygwin path translation +if $cygwin; then + CLASSPATH=`cygpath -p -w "$CLASSPATH"` + HBASE_HOME=`cygpath -d "$HBASE_HOME"` + HBASE_LOG_DIR=`cygpath -d "$HBASE_LOG_DIR"` +fi + +function append_path() { + if [ -z "$1" ]; then + echo $2 + else + echo $1:$2 + fi +} + +JAVA_PLATFORM="" + +#If avail, add Hadoop to the CLASSPATH and to the JAVA_LIBRARY_PATH +HADOOP_IN_PATH=$(PATH="${HADOOP_HOME:-${HADOOP_PREFIX}}/bin:$PATH" which hadoop 2>/dev/null) +if [ -f ${HADOOP_IN_PATH} ]; then + HADOOP_JAVA_LIBRARY_PATH=$(HADOOP_CLASSPATH="$CLASSPATH" ${HADOOP_IN_PATH} \ + org.apache.hadoop.hbase.util.GetJavaProperty java.library.path 2>/dev/null) + if [ -n "$HADOOP_JAVA_LIBRARY_PATH" ]; then + JAVA_LIBRARY_PATH=$(append_path "${JAVA_LIBRARY_PATH}" "$HADOOP_JAVA_LIBRARY_PATH") + fi + CLASSPATH=$(append_path "${CLASSPATH}" `${HADOOP_IN_PATH} classpath 2>/dev/null`) +fi + +if [ -d "${HBASE_HOME}/build/native" -o -d "${HBASE_HOME}/lib/native" ]; then + if [ -z $JAVA_PLATFORM ]; then + JAVA_PLATFORM=`CLASSPATH=${CLASSPATH} ${JAVA} org.apache.hadoop.util.PlatformName | sed -e "s/ /_/g"` + fi + if [ -d "$HBASE_HOME/build/native" ]; then + JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" ${HBASE_HOME}/build/native/${JAVA_PLATFORM}/lib) + fi + + if [ -d "${HBASE_HOME}/lib/native" ]; then + JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" ${HBASE_HOME}/lib/native/${JAVA_PLATFORM}) + fi +fi + +# cygwin path translation +if $cygwin; then + JAVA_LIBRARY_PATH=`cygpath -p "$JAVA_LIBRARY_PATH"` +fi + +# restore ordinary behaviour +unset IFS + +# figure out which class to run +if [ "$COMMAND" = "shell" ] ; then + # eg export JRUBY_HOME=/usr/local/share/jruby + if [ "$JRUBY_HOME" != "" ] ; then + CLASSPATH="$JRUBY_HOME/lib/jruby.jar:$CLASSPATH" + HBASE_OPTS="$HBASE_OPTS -Djruby.home=$JRUBY_HOME -Djruby.lib=$JRUBY_HOME/lib" + fi + CLASS="org.jruby.Main -X+O ${JRUBY_OPTS} ${HBASE_HOME}/bin/hirb.rb" +elif [ "$COMMAND" = "hbck" ] ; then + CLASS='org.apache.hadoop.hbase.util.HBaseFsck' +elif [ "$COMMAND" = "hlog" ] ; then + CLASS='org.apache.hadoop.hbase.regionserver.wal.HLogPrettyPrinter' +elif [ "$COMMAND" = "hfile" ] ; then + CLASS='org.apache.hadoop.hbase.io.hfile.HFile' +elif [ "$COMMAND" = "zkcli" ] ; then + # ZooKeeperMainServerArg returns '-server HOST:PORT' or empty string. + SERVER_ARG=`"$bin"/hbase org.apache.hadoop.hbase.zookeeper.ZooKeeperMainServerArg` + CLASS="org.apache.zookeeper.ZooKeeperMain ${SERVER_ARG}" + +elif [ "$COMMAND" = "master" ] ; then + CLASS='org.apache.hadoop.hbase.master.HMaster' + if [ "$1" != "stop" ] ; then + HBASE_OPTS="$HBASE_OPTS $HBASE_MASTER_OPTS" + fi +elif [ "$COMMAND" = "regionserver" ] ; then + CLASS='org.apache.hadoop.hbase.regionserver.HRegionServer' + if [ "$1" != "stop" ] ; then + HBASE_OPTS="$HBASE_OPTS $HBASE_REGIONSERVER_OPTS" + fi +elif [ "$COMMAND" = "thrift" ] ; then + CLASS='org.apache.hadoop.hbase.thrift.ThriftServer' + if [ "$1" != "stop" ] ; then + HBASE_OPTS="$HBASE_OPTS $HBASE_THRIFT_OPTS" + fi +elif [ "$COMMAND" = "thrift2" ] ; then + CLASS='org.apache.hadoop.hbase.thrift2.ThriftServer' + if [ "$1" != "stop" ] ; then + HBASE_OPTS="$HBASE_OPTS $HBASE_THRIFT_OPTS" + fi +elif [ "$COMMAND" = "rest" ] ; then + CLASS='org.apache.hadoop.hbase.rest.Main' + if [ "$1" != "stop" ] ; then + HBASE_OPTS="$HBASE_OPTS $HBASE_REST_OPTS" + fi +elif [ "$COMMAND" = "avro" ] ; then + CLASS='org.apache.hadoop.hbase.avro.AvroServer' + if [ "$1" != "stop" ] ; then + HBASE_OPTS="$HBASE_OPTS $HBASE_AVRO_OPTS" + fi +elif [ "$COMMAND" = "zookeeper" ] ; then + CLASS='org.apache.hadoop.hbase.zookeeper.HQuorumPeer' + if [ "$1" != "stop" ] ; then + HBASE_OPTS="$HBASE_OPTS $HBASE_ZOOKEEPER_OPTS" + fi + +elif [ "$COMMAND" = "classpath" ] ; then + echo $CLASSPATH + exit 0 +elif [ "$COMMAND" = "version" ] ; then + CLASS='org.apache.hadoop.hbase.util.VersionInfo' +else + CLASS=$COMMAND +fi + +# Have JVM dump heap if we run out of memory. Files will be 'launch directory' +# and are named like the following: java_pid21612.hprof. Apparently it doesn't +# 'cost' to have this flag enabled. Its a 1.6 flag only. See: +# http://blogs.sun.com/alanb/entry/outofmemoryerror_looks_a_bit_better +HBASE_OPTS="$HBASE_OPTS -Dhbase.log.dir=$HBASE_LOG_DIR" +HBASE_OPTS="$HBASE_OPTS -Dhbase.log.file=$HBASE_LOGFILE" +HBASE_OPTS="$HBASE_OPTS -Dhbase.home.dir=$HBASE_HOME" +HBASE_OPTS="$HBASE_OPTS -Dhbase.id.str=$HBASE_IDENT_STRING" +HBASE_OPTS="$HBASE_OPTS -Dhbase.root.logger=${HBASE_ROOT_LOGGER:-INFO,console}" +if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then + HBASE_OPTS="$HBASE_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH" + export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$JAVA_LIBRARY_PATH" +fi + +# Enable security logging on the master and regionserver only +if [ "$COMMAND" = "master" ] || [ "$COMMAND" = "regionserver" ]; then + HBASE_OPTS="$HBASE_OPTS -Dhbase.security.logger=${HBASE_SECURITY_LOGGER:-INFO,DRFAS}" +else + HBASE_OPTS="$HBASE_OPTS -Dhbase.security.logger=${HBASE_SECURITY_LOGGER:-INFO,NullAppender}" +fi + +# Exec unless HBASE_NOEXEC is set. +if [ "${HBASE_NOEXEC}" != "" ]; then + "$JAVA" -XX:OnOutOfMemoryError="kill -9 %p" $JAVA_HEAP_MAX $HBASE_OPTS -classpath "$CLASSPATH" $CLASS "$@" +else + exec "$JAVA" -XX:OnOutOfMemoryError="kill -9 %p" $JAVA_HEAP_MAX $HBASE_OPTS -classpath "$CLASSPATH" $CLASS "$@" +fi diff --git a/titan-accumulo/bin/hbase-config.sh b/titan-accumulo/bin/hbase-config.sh new file mode 100755 index 0000000000..1f25713a94 --- /dev/null +++ b/titan-accumulo/bin/hbase-config.sh @@ -0,0 +1,118 @@ +# +#/** +# * Copyright 2007 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# included in all the hbase scripts with source command +# should not be executable directly +# also should not be passed any arguments, since we need original $* +# Modelled after $HADOOP_HOME/bin/hadoop-env.sh. + +# resolve links - "${BASH_SOURCE-$0}" may be a softlink + +this="${BASH_SOURCE-$0}" +while [ -h "$this" ]; do + ls=`ls -ld "$this"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '.*/.*' > /dev/null; then + this="$link" + else + this=`dirname "$this"`/"$link" + fi +done + +# convert relative path to absolute path +bin=`dirname "$this"` +script=`basename "$this"` +bin=`cd "$bin">/dev/null; pwd` +this="$bin/$script" + +# the root of the hbase installation +if [ -z "$HBASE_HOME" ]; then + export HBASE_HOME=`dirname "$this"`/.. +fi + +#check to see if the conf dir or hbase home are given as an optional arguments +while [ $# -gt 1 ] +do + if [ "--config" = "$1" ] + then + shift + confdir=$1 + shift + HBASE_CONF_DIR=$confdir + elif [ "--hosts" = "$1" ] + then + shift + hosts=$1 + shift + HBASE_REGIONSERVERS=$hosts + else + # Presume we are at end of options and break + break + fi +done + +# Allow alternate hbase conf dir location. +HBASE_CONF_DIR="${HBASE_CONF_DIR:-$HBASE_HOME/conf}" +# List of hbase regions servers. +HBASE_REGIONSERVERS="${HBASE_REGIONSERVERS:-$HBASE_CONF_DIR/regionservers}" +# List of hbase secondary masters. +HBASE_BACKUP_MASTERS="${HBASE_BACKUP_MASTERS:-$HBASE_CONF_DIR/backup-masters}" + +# Source the hbase-env.sh. Will have JAVA_HOME defined. +if [ -f "${HBASE_CONF_DIR}/hbase-env.sh" ]; then + . "${HBASE_CONF_DIR}/hbase-env.sh" +fi + +# Newer versions of glibc use an arena memory allocator that causes virtual +# memory usage to explode. Tune the variable down to prevent vmem explosion. +export MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX:-4} + +if [ -z "$JAVA_HOME" ]; then + for candidate in \ + /usr/lib/jvm/java-6-sun \ + /usr/lib/jvm/java-1.6.0-sun-1.6.0.*/jre \ + /usr/lib/jvm/java-1.6.0-sun-1.6.0.* \ + /usr/lib/j2sdk1.6-sun \ + /usr/java/jdk1.6* \ + /usr/java/jre1.6* \ + /Library/Java/Home ; do + if [ -e $candidate/bin/java ]; then + export JAVA_HOME=$candidate + break + fi + done + # if we didn't set it + if [ -z "$JAVA_HOME" ]; then + cat 1>&2 < http://java.sun.com/javase/downloads/ < | +| | +| HBase requires Java 1.6 or later. | +| NOTE: This script will find Sun Java whether you install using the | +| binary or the RPM based installer. | ++======================================================================+ +EOF + exit 1 + fi +fi diff --git a/titan-accumulo/bin/hbase-daemon.sh b/titan-accumulo/bin/hbase-daemon.sh new file mode 100755 index 0000000000..569bbb3a38 --- /dev/null +++ b/titan-accumulo/bin/hbase-daemon.sh @@ -0,0 +1,202 @@ +#!/usr/bin/env bash +# +#/** +# * Copyright 2007 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# +# Runs a Hadoop hbase command as a daemon. +# +# Environment Variables +# +# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. +# HBASE_LOG_DIR Where log files are stored. PWD by default. +# HBASE_PID_DIR The pid files are stored. /tmp by default. +# HBASE_IDENT_STRING A string representing this instance of hadoop. $USER by default +# HBASE_NICENESS The scheduling priority for daemons. Defaults to 0. +# +# Modelled after $HADOOP_HOME/bin/hadoop-daemon.sh + +usage="Usage: hbase-daemon.sh [--config ]\ + (start|stop|restart) \ + " + +# if no args specified, show usage +if [ $# -le 1 ]; then + echo $usage + exit 1 +fi + +bin=`dirname "${BASH_SOURCE-$0}"` +bin=`cd "$bin">/dev/null; pwd` + +. "$bin"/hbase-config.sh + +# get arguments +startStop=$1 +shift + +command=$1 +shift + +hbase_rotate_log () +{ + log=$1; + num=5; + if [ -n "$2" ]; then + num=$2 + fi + if [ -f "$log" ]; then # rotate logs + while [ $num -gt 1 ]; do + prev=`expr $num - 1` + [ -f "$log.$prev" ] && mv -f "$log.$prev" "$log.$num" + num=$prev + done + mv -f "$log" "$log.$num"; + fi +} + +wait_until_done () +{ + p=$1 + cnt=${HBASE_SLAVE_TIMEOUT:-300} + origcnt=$cnt + while kill -0 $p > /dev/null 2>&1; do + if [ $cnt -gt 1 ]; then + cnt=`expr $cnt - 1` + sleep 1 + else + echo "Process did not complete after $origcnt seconds, killing." + kill -9 $p + exit 1 + fi + done + return 0 +} + +# get log directory +if [ "$HBASE_LOG_DIR" = "" ]; then + export HBASE_LOG_DIR="$HBASE_HOME/logs" +fi +mkdir -p "$HBASE_LOG_DIR" + +if [ "$HBASE_PID_DIR" = "" ]; then + HBASE_PID_DIR=/tmp +fi + +if [ "$HBASE_IDENT_STRING" = "" ]; then + export HBASE_IDENT_STRING="$USER" +fi + +# Some variables +# Work out java location so can print version into log. +if [ "$JAVA_HOME" != "" ]; then + #echo "run java in $JAVA_HOME" + JAVA_HOME=$JAVA_HOME +fi +if [ "$JAVA_HOME" = "" ]; then + echo "Error: JAVA_HOME is not set." + exit 1 +fi +JAVA=$JAVA_HOME/bin/java +export HBASE_LOG_PREFIX=hbase-$HBASE_IDENT_STRING-$command-$HOSTNAME +export HBASE_LOGFILE=$HBASE_LOG_PREFIX.log +export HBASE_ROOT_LOGGER="INFO,DRFA" +export HBASE_SECURITY_LOGGER="INFO,DRFAS" +logout=$HBASE_LOG_DIR/$HBASE_LOG_PREFIX.out +loggc=$HBASE_LOG_DIR/$HBASE_LOG_PREFIX.gc +loglog="${HBASE_LOG_DIR}/${HBASE_LOGFILE}" +pid=$HBASE_PID_DIR/hbase-$HBASE_IDENT_STRING-$command.pid + +if [ "$HBASE_USE_GC_LOGFILE" = "true" ]; then + export HBASE_GC_OPTS=" -Xloggc:${loggc}" +fi + +# Set default scheduling priority +if [ "$HBASE_NICENESS" = "" ]; then + export HBASE_NICENESS=0 +fi + +case $startStop in + + (start) + mkdir -p "$HBASE_PID_DIR" + if [ -f $pid ]; then + if kill -0 `cat $pid` > /dev/null 2>&1; then + echo $command running as process `cat $pid`. Stop it first. + exit 1 + fi + fi + + hbase_rotate_log $logout + hbase_rotate_log $loggc + echo starting $command, logging to $logout + # Add to the command log file vital stats on our environment. + echo "`date` Starting $command on `hostname`" >> $loglog + echo "`ulimit -a`" >> $loglog 2>&1 + nohup nice -n $HBASE_NICENESS "$HBASE_HOME"/bin/hbase \ + --config "${HBASE_CONF_DIR}" \ + $command "$@" $startStop > "$logout" 2>&1 < /dev/null & + echo $! > $pid + sleep 1; head "$logout" + ;; + + (stop) + if [ -f $pid ]; then + # kill -0 == see if the PID exists + if kill -0 `cat $pid` > /dev/null 2>&1; then + echo -n stopping $command + echo "`date` Terminating $command" >> $loglog + kill `cat $pid` > /dev/null 2>&1 + while kill -0 `cat $pid` > /dev/null 2>&1; do + echo -n "." + sleep 1; + done + rm $pid + echo + else + retval=$? + echo no $command to stop because kill -0 of pid `cat $pid` failed with status $retval + fi + else + echo no $command to stop because no pid file $pid + fi + ;; + + (restart) + thiscmd=$0 + args=$@ + # stop the command + $thiscmd --config "${HBASE_CONF_DIR}" stop $command $args & + wait_until_done $! + # wait a user-specified sleep period + sp=${HBASE_RESTART_SLEEP:-3} + if [ $sp -gt 0 ]; then + sleep $sp + fi + # start the command + $thiscmd --config "${HBASE_CONF_DIR}" start $command $args & + wait_until_done $! + ;; + + (*) + echo $usage + exit 1 + ;; + +esac diff --git a/titan-accumulo/bin/hbase-daemons.sh b/titan-accumulo/bin/hbase-daemons.sh new file mode 100755 index 0000000000..843eaaa74f --- /dev/null +++ b/titan-accumulo/bin/hbase-daemons.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# +#/** +# * Copyright 2007 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# +# Run a hbase command on all slave hosts. +# Modelled after $HADOOP_HOME/bin/hadoop-daemons.sh + +usage="Usage: hbase-daemons.sh [--config ] \ + [--hosts regionserversfile] [start|stop] command args..." + +# if no args specified, show usage +if [ $# -le 1 ]; then + echo $usage + exit 1 +fi + +bin=`dirname "${BASH_SOURCE-$0}"` +bin=`cd "$bin">/dev/null; pwd` + +. $bin/hbase-config.sh + +remote_cmd="cd ${HBASE_HOME}; $bin/hbase-daemon.sh --config ${HBASE_CONF_DIR} $@" +args="--hosts ${HBASE_REGIONSERVERS} --config ${HBASE_CONF_DIR} $remote_cmd" + +command=$2 +case $command in + (zookeeper) + exec "$bin/zookeepers.sh" $args + ;; + (master-backup) + exec "$bin/master-backup.sh" $args + ;; + (*) + exec "$bin/regionservers.sh" $args + ;; +esac + diff --git a/titan-accumulo/bin/local-master-backup.sh b/titan-accumulo/bin/local-master-backup.sh new file mode 100755 index 0000000000..2c0a4c02c7 --- /dev/null +++ b/titan-accumulo/bin/local-master-backup.sh @@ -0,0 +1,55 @@ +#!/bin/sh +#/** +# * Copyright 2007 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# This is used for starting multiple masters on the same machine. +# run it from hbase-dir/ just like 'bin/hbase' +# Supports up to 10 masters (limitation = overlapping ports) + +bin=`dirname "${BASH_SOURCE-$0}"` +bin=`cd "$bin" >/dev/null && pwd` + +if [ $# -lt 2 ]; then + S=`basename "${BASH_SOURCE-$0}"` + echo "Usage: $S [start|stop] offset(s)" + echo "" + echo " e.g. $S start 1" + exit +fi + +# sanity check: make sure your master opts don't use ports [i.e. JMX/DBG] +export HBASE_MASTER_OPTS=" " + +run_master () { + DN=$2 + export HBASE_IDENT_STRING="$USER-$DN" + HBASE_MASTER_ARGS="\ + -D hbase.master.port=`expr 60000 + $DN` \ + -D hbase.master.info.port=`expr 60010 + $DN` \ + --backup" + "$bin"/hbase-daemon.sh $1 master $HBASE_MASTER_ARGS +} + +cmd=$1 +shift; + +for i in $* +do + run_master $cmd $i +done diff --git a/titan-accumulo/bin/local-regionservers.sh b/titan-accumulo/bin/local-regionservers.sh new file mode 100755 index 0000000000..a4d5a1d932 --- /dev/null +++ b/titan-accumulo/bin/local-regionservers.sh @@ -0,0 +1,54 @@ +#!/bin/sh +#/** +# * Copyright 2007 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# This is used for starting multiple regionservers on the same machine. +# run it from hbase-dir/ just like 'bin/hbase' +# Supports up to 100 regionservers (limitation = overlapping ports) + +bin=`dirname "${BASH_SOURCE-$0}"` +bin=`cd "$bin" >/dev/null && pwd` + +if [ $# -lt 2 ]; then + S=`basename "${BASH_SOURCE-$0}"` + echo "Usage: $S [start|stop] offset(s)" + echo "" + echo " e.g. $S start 1 2" + exit +fi + +# sanity check: make sure your regionserver opts don't use ports [i.e. JMX/DBG] +export HBASE_REGIONSERVER_OPTS=" " + +run_regionserver () { + DN=$2 + export HBASE_IDENT_STRING="$USER-$DN" + HBASE_REGIONSERVER_ARGS="\ + -D hbase.regionserver.port=`expr 60200 + $DN` \ + -D hbase.regionserver.info.port=`expr 60300 + $DN`" + "$bin"/hbase-daemon.sh $1 regionserver $HBASE_REGIONSERVER_ARGS +} + +cmd=$1 +shift; + +for i in $* +do + run_regionserver $cmd $i +done diff --git a/titan-accumulo/bin/master-backup.sh b/titan-accumulo/bin/master-backup.sh new file mode 100755 index 0000000000..114757c763 --- /dev/null +++ b/titan-accumulo/bin/master-backup.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# +#/** +# * Copyright 2010 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# +# Run a shell command on all backup master hosts. +# +# Environment Variables +# +# HBASE_BACKUP_MASTERS File naming remote hosts. +# Default is ${HBASE_CONF_DIR}/backup-masters +# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. +# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. +# HADOOP_SLAVE_SLEEP Seconds to sleep between spawning remote commands. +# HADOOP_SSH_OPTS Options passed to ssh when running remote commands. +# +# Modelled after $HADOOP_HOME/bin/slaves.sh. + +usage="Usage: $0 [--config ] command..." + +# if no args specified, show usage +if [ $# -le 0 ]; then + echo $usage + exit 1 +fi + +bin=`dirname "${BASH_SOURCE-$0}"` +bin=`cd "$bin">/dev/null; pwd` + +. "$bin"/hbase-config.sh + +# If the master backup file is specified in the command line, +# then it takes precedence over the definition in +# hbase-env.sh. Save it here. +HOSTLIST=$HBASE_BACKUP_MASTERS + +if [ "$HOSTLIST" = "" ]; then + if [ "$HBASE_BACKUP_MASTERS" = "" ]; then + export HOSTLIST="${HBASE_CONF_DIR}/backup-masters" + else + export HOSTLIST="${HBASE_BACKUP_MASTERS}" + fi +fi + + +args=${@// /\\ } +args=${args/master-backup/master} + +if [ -f $HOSTLIST ]; then + for hmaster in `cat "$HOSTLIST"`; do + ssh $HBASE_SSH_OPTS $hmaster $"$args --backup" \ + 2>&1 | sed "s/^/$hmaster: /" & + if [ "$HBASE_SLAVE_SLEEP" != "" ]; then + sleep $HBASE_SLAVE_SLEEP + fi + done +fi + +wait diff --git a/titan-accumulo/bin/regionservers.sh b/titan-accumulo/bin/regionservers.sh new file mode 100755 index 0000000000..9759f2b00c --- /dev/null +++ b/titan-accumulo/bin/regionservers.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# +#/** +# * Copyright 2007 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# +# Run a shell command on all regionserver hosts. +# +# Environment Variables +# +# HBASE_REGIONSERVERS File naming remote hosts. +# Default is ${HADOOP_CONF_DIR}/regionservers +# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. +# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. +# HADOOP_SLAVE_SLEEP Seconds to sleep between spawning remote commands. +# HADOOP_SSH_OPTS Options passed to ssh when running remote commands. +# +# Modelled after $HADOOP_HOME/bin/slaves.sh. + +usage="Usage: regionservers [--config ] command..." + +# if no args specified, show usage +if [ $# -le 0 ]; then + echo $usage + exit 1 +fi + +bin=`dirname "${BASH_SOURCE-$0}"` +bin=`cd "$bin">/dev/null; pwd` + +. "$bin"/hbase-config.sh + +# If the regionservers file is specified in the command line, +# then it takes precedence over the definition in +# hbase-env.sh. Save it here. +HOSTLIST=$HBASE_REGIONSERVERS + +if [ "$HOSTLIST" = "" ]; then + if [ "$HBASE_REGIONSERVERS" = "" ]; then + export HOSTLIST="${HBASE_CONF_DIR}/regionservers" + else + export HOSTLIST="${HBASE_REGIONSERVERS}" + fi +fi + +for regionserver in `cat "$HOSTLIST"`; do + if ${HBASE_SLAVE_PARALLEL:-true}; then + ssh $HBASE_SSH_OPTS $regionserver $"${@// /\\ }" \ + 2>&1 | sed "s/^/$regionserver: /" & + else # run each command serially + ssh $HBASE_SSH_OPTS $regionserver $"${@// /\\ }" \ + 2>&1 | sed "s/^/$regionserver: /" + fi + if [ "$HBASE_SLAVE_SLEEP" != "" ]; then + sleep $HBASE_SLAVE_SLEEP + fi +done + +wait diff --git a/titan-accumulo/bin/rolling-restart.sh b/titan-accumulo/bin/rolling-restart.sh new file mode 100755 index 0000000000..07a78c71ed --- /dev/null +++ b/titan-accumulo/bin/rolling-restart.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash +# +#/** +# * Copyright 2007 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# +# Run a shell command on all regionserver hosts. +# +# Environment Variables +# +# HBASE_REGIONSERVERS File naming remote hosts. +# Default is ${HADOOP_CONF_DIR}/regionservers +# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. +# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. +# HADOOP_SLAVE_SLEEP Seconds to sleep between spawning remote commands. +# HADOOP_SLAVE_TIMEOUT Seconds to wait for timing out a remote command. +# HADOOP_SSH_OPTS Options passed to ssh when running remote commands. +# +# Modelled after $HADOOP_HOME/bin/slaves.sh. + +usage="Usage: $0 [--config ] [--rs-only] [--master-only] [--graceful]" + +bin=`dirname "$0"` +bin=`cd "$bin">/dev/null; pwd` + +. "$bin"/hbase-config.sh + +# start hbase daemons +errCode=$? +if [ $errCode -ne 0 ] +then + exit $errCode +fi + +function usage() { + echo $usage + exit 1 +} + + +RR_RS=1 +RR_MASTER=1 +RR_GRACEFUL=0 + +for x in "$@" ; do + case "$x" in + --rs-only|-r) + RR_RS=1 + RR_MASTER=0 + RR_GRACEFUL=0 + ;; + --master-only) + RR_RS=0 + RR_MASTER=1 + RR_GRACEFUL=0 + ;; + --graceful) + RR_RS=0 + RR_MASTER=0 + RR_GRACEFUL=1 + ;; + *) + echo Bad argument: $x + usage + exit 1 + ;; + esac +done + +# quick function to get a value from the HBase config file +# HBASE-6504 - only take the first line of the output in case verbose gc is on +distMode=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed | head -n 1` +if [ "$distMode" == 'false' ]; then + if [ $RR_RS -ne 1 ] || [ $RR_MASTER -ne 1 ]; then + echo Cant do selective rolling restart if not running distributed + exit 1 + fi + "$bin"/hbase-daemon.sh restart master +else + zparent=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.parent` + if [ "$zparent" == "null" ]; then zparent="/hbase"; fi + + if [ $RR_MASTER -eq 1 ]; then + # stop all masters before re-start to avoid races for master znode + "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" stop master + "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ + --hosts "${HBASE_BACKUP_MASTERS}" stop master-backup + + # make sure the master znode has been deleted before continuing + zmaster=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.master` + if [ "$zmaster" == "null" ]; then zmaster="master"; fi + zmaster=$zparent/$zmaster + echo -n "Waiting for Master ZNode ${zmaster} to expire" + while ! bin/hbase zkcli stat $zmaster 2>&1 | grep "Node does not exist"; do + echo -n "." + sleep 1 + done + echo #force a newline + + # all masters are down, now restart + "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" start master + "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ + --hosts "${HBASE_BACKUP_MASTERS}" start master-backup + + echo "Wait a minute for master to come up join cluster" + sleep 60 + + # Master joing cluster will start in cleaning out regions in transition. + # Wait until the master has cleaned out regions in transition before + # giving it a bunch of work to do; master is vulnerable during startup + zunassigned=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.unassigned` + if [ "$zunassigned" == "null" ]; then zunassigned="unassigned"; fi + zunassigned="$zparent/$zunassigned" + echo -n "Waiting for ${zunassigned} to empty" + while true ; do + unassigned=`$bin/hbase zkcli stat ${zunassigned} 2>&1 |grep -e 'numChildren = '|sed -e 's,numChildren = ,,'` + if test 0 -eq ${unassigned} + then + break + else + echo -n " ${unassigned}" + fi + sleep 1 + done + fi + + if [ $RR_RS -eq 1 ]; then + # unlike the masters, roll all regionservers one-at-a-time + export HBASE_SLAVE_PARALLEL=false + "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ + --hosts "${HBASE_REGIONSERVERS}" restart regionserver + fi + + if [ $RR_GRACEFUL -eq 1 ]; then + # gracefully restart all online regionservers + online_regionservers=`$bin/hbase zkcli ls $zparent/rs 2>&1 | tail -1 | sed "s/\[//" | sed "s/\]//"` + for rs in $online_regionservers + do + rs_parts=(${rs//,/ }) + hostname=${rs_parts[0]} + echo "Gracefully restarting: $hostname" + "$bin"/graceful_stop.sh --config "${HBASE_CONF_DIR}" --restart --reload --debug "$hostname" + sleep 1 + done + fi +fi diff --git a/titan-accumulo/bin/start-hbase.sh b/titan-accumulo/bin/start-hbase.sh new file mode 100755 index 0000000000..0b0ce1a58d --- /dev/null +++ b/titan-accumulo/bin/start-hbase.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# +#/** +# * Copyright 2007 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# Modelled after $HADOOP_HOME/bin/start-hbase.sh. + +# Start hadoop hbase daemons. +# Run this on master node. +usage="Usage: start-hbase.sh" + +bin=`dirname "${BASH_SOURCE-$0}"` +bin=`cd "$bin">/dev/null; pwd` + +. "$bin"/hbase-config.sh + +# start hbase daemons +errCode=$? +if [ $errCode -ne 0 ] +then + exit $errCode +fi +# HBASE-6504 - only take the first line of the output in case verbose gc is on +distMode=`$bin/hbase --config "$HBASE_CONF_DIR" org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed | head -n 1` + + +if [ "$distMode" == 'false' ] +then + "$bin"/hbase-daemon.sh start master +else + "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" start zookeeper + "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" start master + "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ + --hosts "${HBASE_REGIONSERVERS}" start regionserver + "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ + --hosts "${HBASE_BACKUP_MASTERS}" start master-backup +fi diff --git a/titan-accumulo/bin/stop-hbase.sh b/titan-accumulo/bin/stop-hbase.sh new file mode 100755 index 0000000000..e9749fee28 --- /dev/null +++ b/titan-accumulo/bin/stop-hbase.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# +#/** +# * Copyright 2007 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# Modelled after $HADOOP_HOME/bin/stop-hbase.sh. + +# Stop hadoop hbase daemons. Run this on master node. + +bin=`dirname "${BASH_SOURCE-$0}"` +bin=`cd "$bin">/dev/null; pwd` + +. "$bin"/hbase-config.sh + +# variables needed for stop command +if [ "$HBASE_LOG_DIR" = "" ]; then + export HBASE_LOG_DIR="$HBASE_HOME/logs" +fi +mkdir -p "$HBASE_LOG_DIR" + +if [ "$HBASE_IDENT_STRING" = "" ]; then + export HBASE_IDENT_STRING="$USER" +fi + +export HBASE_LOG_PREFIX=hbase-$HBASE_IDENT_STRING-master-$HOSTNAME +export HBASE_LOGFILE=$HBASE_LOG_PREFIX.log +logout=$HBASE_LOG_DIR/$HBASE_LOG_PREFIX.out +loglog="${HBASE_LOG_DIR}/${HBASE_LOGFILE}" +pid=${HBASE_PID_DIR:-/tmp}/hbase-$HBASE_IDENT_STRING-master.pid + +echo -n stopping hbase +echo "`date` Stopping hbase (via master)" >> $loglog + +nohup nice -n ${HBASE_NICENESS:-0} "$HBASE_HOME"/bin/hbase \ + --config "${HBASE_CONF_DIR}" \ + master stop "$@" > "$logout" 2>&1 < /dev/null & + +while kill -0 `cat $pid` > /dev/null 2>&1; do + echo -n "." + sleep 1; +done +# Add a CR after we're done w/ dots. +echo + +# distributed == false means that the HMaster will kill ZK when it exits +# HBASE-6504 - only take the first line of the output in case verbose gc is on +distMode=`$bin/hbase --config "$HBASE_CONF_DIR" org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed | head -n 1` +if [ "$distMode" == 'true' ] +then + # TODO: store backup masters in ZooKeeper and have the primary send them a shutdown message + # stop any backup masters + "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ + --hosts "${HBASE_BACKUP_MASTERS}" stop master-backup + + "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" stop zookeeper +fi diff --git a/titan-accumulo/bin/zookeepers.sh b/titan-accumulo/bin/zookeepers.sh new file mode 100755 index 0000000000..89a214e5a8 --- /dev/null +++ b/titan-accumulo/bin/zookeepers.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# +#/** +# * Copyright 2009 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# +# Run a shell command on all zookeeper hosts. +# +# Environment Variables +# +# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. +# HBASE_SLAVE_SLEEP Seconds to sleep between spawning remote commands. +# HBASE_SSH_OPTS Options passed to ssh when running remote commands. +# +# Modelled after $HADOOP_HOME/bin/slaves.sh. + +usage="Usage: zookeepers [--config ] command..." + +# if no args specified, show usage +if [ $# -le 0 ]; then + echo $usage + exit 1 +fi + +bin=`dirname "${BASH_SOURCE-$0}"` +bin=`cd "$bin">/dev/null; pwd` + +. "$bin"/hbase-config.sh + +if [ "$HBASE_MANAGES_ZK" = "" ]; then + HBASE_MANAGES_ZK=true +fi + +if [ "$HBASE_MANAGES_ZK" = "true" ]; then + hosts=`"$bin"/hbase org.apache.hadoop.hbase.zookeeper.ZKServerTool | grep '^ZK host:' | sed 's,^ZK host:,,'` + cmd=$"${@// /\\ }" + for zookeeper in $hosts; do + ssh $HBASE_SSH_OPTS $zookeeper $cmd 2>&1 | sed "s/^/$zookeeper: /" & + if [ "$HBASE_SLAVE_SLEEP" != "" ]; then + sleep $HBASE_SLAVE_SLEEP + fi + done +fi + +wait diff --git a/titan-accumulo/config/hadoop-metrics.properties b/titan-accumulo/config/hadoop-metrics.properties new file mode 100644 index 0000000000..2060e22df5 --- /dev/null +++ b/titan-accumulo/config/hadoop-metrics.properties @@ -0,0 +1,70 @@ +# See http://wiki.apache.org/hadoop/GangliaMetrics +# Make sure you know whether you are using ganglia 3.0 or 3.1. +# If 3.1, you will have to patch your hadoop instance with HADOOP-4675 +# And, yes, this file is named hadoop-metrics.properties rather than +# hbase-metrics.properties because we're leveraging the hadoop metrics +# package and hadoop-metrics.properties is an hardcoded-name, at least +# for the moment. +# +# See also http://hadoop.apache.org/hbase/docs/current/metrics.html +# GMETADHOST_IP is the hostname (or) IP address of the server on which the ganglia +# meta daemon (gmetad) service is running + +# Configuration of the "hbase" context for NullContextWithUpdateThread +# NullContextWithUpdateThread is a null context which has a thread calling +# periodically when monitoring is started. This keeps the data sampled +# correctly. +hbase.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread +hbase.period=10 + +# Configuration of the "hbase" context for file +# hbase.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext +# hbase.fileName=/tmp/metrics_hbase.log + +# HBase-specific configuration to reset long-running stats (e.g. compactions) +# If this variable is left out, then the default is no expiration. +hbase.extendedperiod = 3600 + +# Configuration of the "hbase" context for ganglia +# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) +# hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext +# hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 +# hbase.period=10 +# hbase.servers=GMETADHOST_IP:8649 + +# Configuration of the "jvm" context for null +jvm.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread +jvm.period=10 + +# Configuration of the "jvm" context for file +# jvm.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext +# jvm.fileName=/tmp/metrics_jvm.log + +# Configuration of the "jvm" context for ganglia +# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) +# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext +# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 +# jvm.period=10 +# jvm.servers=GMETADHOST_IP:8649 + +# Configuration of the "rpc" context for null +rpc.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread +rpc.period=10 + +# Configuration of the "rpc" context for file +# rpc.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext +# rpc.fileName=/tmp/metrics_rpc.log + +# Configuration of the "rpc" context for ganglia +# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) +# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext +# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 +# rpc.period=10 +# rpc.servers=GMETADHOST_IP:8649 + +# Configuration of the "rest" context for ganglia +# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) +# rest.class=org.apache.hadoop.metrics.ganglia.GangliaContext +# rest.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 +# rest.period=10 +# rest.servers=GMETADHOST_IP:8649 diff --git a/titan-accumulo/config/hbase-env.sh b/titan-accumulo/config/hbase-env.sh new file mode 100644 index 0000000000..4590184793 --- /dev/null +++ b/titan-accumulo/config/hbase-env.sh @@ -0,0 +1,94 @@ +# +#/** +# * Copyright 2007 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# Set environment variables here. + +# The java implementation to use. Java 1.6 required. +# export JAVA_HOME=/usr/java/jdk1.6.0/ + +# Extra Java CLASSPATH elements. Optional. +export HBASE_CLASSPATH=$HBASE_CLASSPATH:./target/test-lib + +# The maximum amount of heap to use, in MB. Default is 1000. +# export HBASE_HEAPSIZE=1000 + +# Extra Java runtime options. +# Below are what we set by default. May only work with SUN JVM. +# For more on why as well as other possible settings, +# see http://wiki.apache.org/hadoop/PerformanceTuning +export HBASE_OPTS="$HBASE_OPTS -XX:+UseConcMarkSweepGC" + +# Uncomment below to enable java garbage collection logging in the .out file. +# export HBASE_OPTS="$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps $HBASE_GC_OPTS" + +# Uncomment below (along with above GC logging) to put GC information in its own logfile (will set HBASE_GC_OPTS) +# export HBASE_USE_GC_LOGFILE=true + + +# Uncomment below if you intend to use the EXPERIMENTAL off heap cache. +# export HBASE_OPTS="$HBASE_OPTS -XX:MaxDirectMemorySize=" +# Set hbase.offheapcache.percentage in hbase-site.xml to a nonzero value. + + +# Uncomment and adjust to enable JMX exporting +# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access. +# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html +# +# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" +# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10101" +# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10102" +# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103" +# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104" + +# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default. +# export HBASE_REGIONSERVERS=${HBASE_HOME}/conf/regionservers + +# File naming hosts on which backup HMaster will run. $HBASE_HOME/conf/backup-masters by default. +# export HBASE_BACKUP_MASTERS=${HBASE_HOME}/conf/backup-masters + +# Extra ssh options. Empty by default. +# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR" + +# Where log files are stored. $HBASE_HOME/logs by default. +# export HBASE_LOG_DIR=${HBASE_HOME}/logs + +# Enable remote JDWP debugging of major HBase processes. Meant for Core Developers +# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8070" +# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8071" +# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8072" +# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8073" + +# A string representing this instance of hbase. $USER by default. +# export HBASE_IDENT_STRING=$USER + +# The scheduling priority for daemon processes. See 'man nice'. +# export HBASE_NICENESS=10 + +# The directory where pid files are stored. /tmp by default. +# export HBASE_PID_DIR=/var/hadoop/pids + +# Seconds to sleep between slave commands. Unset by default. This +# can be useful in large clusters, where, e.g., slave rsyncs can +# otherwise arrive faster than the master can service them. +# export HBASE_SLAVE_SLEEP=0.1 + +# Tell HBase whether it should manage it's own instance of Zookeeper or not. +# export HBASE_MANAGES_ZK=true diff --git a/titan-accumulo/config/hbase-policy.xml b/titan-accumulo/config/hbase-policy.xml new file mode 100644 index 0000000000..e45f23c962 --- /dev/null +++ b/titan-accumulo/config/hbase-policy.xml @@ -0,0 +1,53 @@ + + + + + + + security.client.protocol.acl + * + ACL for HRegionInterface protocol implementations (ie. + clients talking to HRegionServers) + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed. + + + + security.admin.protocol.acl + * + ACL for HMasterInterface protocol implementation (ie. + clients talking to HMaster for admin operations). + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed. + + + + security.masterregion.protocol.acl + * + ACL for HMasterRegionInterface protocol implementations + (for HRegionServers communicating with HMaster) + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed. + + diff --git a/titan-accumulo/config/hbase-site.xml b/titan-accumulo/config/hbase-site.xml new file mode 100644 index 0000000000..a0587829a1 --- /dev/null +++ b/titan-accumulo/config/hbase-site.xml @@ -0,0 +1,37 @@ + + + + + + hbase.rootdir + file:///tmp/hbase + + + hbase.zookeeper.property.dataDir + /tmp/zookeeper + + + diff --git a/titan-accumulo/config/log4j.properties b/titan-accumulo/config/log4j.properties new file mode 100644 index 0000000000..f6c3305178 --- /dev/null +++ b/titan-accumulo/config/log4j.properties @@ -0,0 +1,74 @@ +# Define some default values that can be overridden by system properties +hbase.root.logger=INFO,console +hbase.security.logger=INFO,console +hbase.log.dir=. +hbase.log.file=hbase.log + +# Define the root logger to the system property "hbase.root.logger". +log4j.rootLogger=${hbase.root.logger} + +# Logging Threshold +log4j.threshold=ALL + +# +# Daily Rolling File Appender +# +log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender +log4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file} + +# Rollver at midnight +log4j.appender.DRFA.DatePattern=.yyyy-MM-dd + +# 30-day backup +#log4j.appender.DRFA.MaxBackupIndex=30 +log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout + +# Pattern format: Date LogLevel LoggerName LogMessage +log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n + +# Debugging Pattern format +#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n + +# +# Security audit appender +# +hbase.security.log.file=SecurityAuth.audit +log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender +log4j.appender.DRFAS.File=${hbase.log.dir}/${hbase.security.log.file} +log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout +log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +log4j.category.SecurityLogger=${hbase.security.logger} +log4j.additivity.SecurityLogger=false + +# +# Null Appender +# +log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender + +# +# console +# Add "console" to rootlogger above if you want to use this +# +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n + +# Custom Logging levels + +log4j.logger.org.apache.zookeeper=INFO +#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG +log4j.logger.org.apache.hadoop.hbase=DEBUG +# Make these two classes INFO-level. Make them DEBUG to see more zk debug. +log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=INFO +log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher=INFO +#log4j.logger.org.apache.hadoop.dfs=DEBUG +# Set this class to log INFO only otherwise its OTT + +# Uncomment this line to enable tracing on _every_ RPC call (this can be a lot of output) +#log4j.logger.org.apache.hadoop.ipc.HBaseServer.trace=DEBUG + +# Uncomment the below if you want to remove logging of client region caching' +# and scan of .META. messages +# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=INFO +# log4j.logger.org.apache.hadoop.hbase.client.MetaScanner=INFO diff --git a/titan-accumulo/config/regionservers b/titan-accumulo/config/regionservers new file mode 100644 index 0000000000..2fbb50c4a8 --- /dev/null +++ b/titan-accumulo/config/regionservers @@ -0,0 +1 @@ +localhost diff --git a/titan-accumulo/config/whirr-hbase.properties b/titan-accumulo/config/whirr-hbase.properties new file mode 100644 index 0000000000..a9ced05fb7 --- /dev/null +++ b/titan-accumulo/config/whirr-hbase.properties @@ -0,0 +1,36 @@ +# A Whirr Receipe for HBase + Hadoop +# Read the Configuration Guide for more info: +# http://whirr.apache.org/docs/latest/configuration-guide.html + +# Change the cluster name here +whirr.cluster-name=hbase-hadoop + +# Change the number of machines in the cluster here +whirr.instance-templates=1 zookeeper+hadoop-namenode+hadoop-jobtracker+hbase-master,3 hadoop-datanode+hadoop-tasktracker+hbase-regionserver + +# replication level should not be higher than number of data nodes +hbase-site.dfs.replication=2 + +# For EC2 set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables. +whirr.provider=aws-ec2 +whirr.identity=${env:AWS_ACCESS_KEY_ID} +whirr.credential=${env:AWS_SECRET_ACCESS_KEY} + +# The size of the instance to use. See http://aws.amazon.com/ec2/instance-types/ +whirr.hardware-id=m1.large +# Ubuntu 10.04 LTS Lucid. See http://alestic.com/ +whirr.image-id=us-east-1/ami-da0cf8b3 +# If you choose a different location, make sure whirr.image-id is updated too +whirr.location-id=us-east-1 + +# By default use the user system SSH keys. Override them here. +# whirr.private-key-file=${sys:user.home}/.ssh/id_rsa +# whirr.public-key-file=${whirr.private-key-file}.pub + +whirr.hbase.tarball.url=http://archive.apache.org/dist/hbase/hbase-0.94.1/hbase-0.94.1.tar.gz +whirr.hadoop.tarball.url=http://archive.apache.org/dist/hadoop/core/hadoop-1.0.3/hadoop-1.0.3.tar.gz + +# Options for the hbase master & regionserver processes +#hbase-env.HBASE_MASTER_OPTS=-Xms1000m -Xmx1000m -Xmn256m -XX:+UseConcMarkSweepGC -XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -Xloggc:/data/hbase/logs/hbase-master-gc.log +#hbase-env.HBASE_REGIONSERVER_OPTS=-Xms2000m -Xmx2000m -Xmn256m -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=88 -XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -Xloggc:/data/hbase/logs/hbase-regionserver-gc.log + diff --git a/titan-accumulo/pom.xml b/titan-accumulo/pom.xml new file mode 100644 index 0000000000..ec64aca992 --- /dev/null +++ b/titan-accumulo/pom.xml @@ -0,0 +1,154 @@ + + 4.0.0 + + com.thinkaurelius.titan + titan + 0.3.1 + ../pom.xml + + titan-accumulo + Titan-Accumulo: Distributed Graph Database + http://thinkaurelius.github.com/titan/ + + + com.thinkaurelius.titan + titan-core + ${titan.version} + + + com.thinkaurelius.titan + titan-test + ${titan.version} + test + + + com.thinkaurelius.titan + titan-es + ${titan.version} + + + org.apache.hbase + hbase + + 0.94.1 + + + avro + org.apache.avro + + + jruby-complete + org.jruby + + + + + org.apache.hadoop + hadoop-core + + 1.0.3 + + + + + comprehensive + + + + maven-surefire-plugin + 2.12.1 + + -Xms256m -Xmx1024m + + **/*Test.java + + + **/Internal*.java + **/*Suite.java + + always + + + + + + + + ${basedir}/target + ${project.artifactId}-${project.version} + + + ${basedir}/src/main/resources + true + + + + + ${basedir}/src/test/resources + + + + + + maven-dependency-plugin + 2.7 + + + test-compile + + copy-dependencies + + + target/test-lib + false + false + true + + + + + + maven-surefire-plugin + 2.12.1 + + -Xms256m -Xmx756m + + + **/*PerformanceTest.java + **/*ConcurrentTest.java + + + + + maven-assembly-plugin + 2.4 + + + package + + attached + + + + + false + + src/assembly/distribution.xml + + ${project.artifactId}-${project.version} + target + target/assembly/work + warn + + 420 + + 493 + + 493 + + + + + + \ No newline at end of file diff --git a/titan-accumulo/src/assembly/distribution.xml b/titan-accumulo/src/assembly/distribution.xml new file mode 100644 index 0000000000..3004a3caeb --- /dev/null +++ b/titan-accumulo/src/assembly/distribution.xml @@ -0,0 +1,100 @@ + + distribution + + zip + + + + + 0775 + ../titan-all/src/main/bin + bin + + *.sh + *.bat + + + + + 0775 + ./bin + bin + + *.sh + *.bat + + + + + ../bin/hbase.local + + + + ./config + config + + + src + + + ../doc + + + ../config + + + ../target/site/apidocs + doc/javadoc + + + target/*.jar + /lib + + + ../bin + bin + + hbase* + + + + + + pom.xml + src + + + ../LICENSE.txt + / + + + ../CHANGELOG.textile + / + + + ../UPGRADE.textile + / + + + ../NOTICE.txt + / + + + ../titan-all/src/main/bin/README.txt + / + + + + + + /lib + false + compile + + + /lib + false + provided + + + diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseKeyColumnValueStore.java b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseKeyColumnValueStore.java new file mode 100644 index 0000000000..f2fc9d5bf0 --- /dev/null +++ b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseKeyColumnValueStore.java @@ -0,0 +1,307 @@ +package com.thinkaurelius.titan.diskstorage.hbase; + +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import com.google.common.collect.Iterators; +import com.thinkaurelius.titan.diskstorage.PermanentStorageException; +import com.thinkaurelius.titan.diskstorage.StaticBuffer; +import com.thinkaurelius.titan.diskstorage.StorageException; +import com.thinkaurelius.titan.diskstorage.TemporaryStorageException; +import com.thinkaurelius.titan.diskstorage.keycolumnvalue.*; +import com.thinkaurelius.titan.diskstorage.util.RecordIterator; +import com.thinkaurelius.titan.diskstorage.util.StaticArrayBuffer; +import com.thinkaurelius.titan.util.system.IOUtils; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.filter.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; +import java.io.IOException; +import java.util.*; + +/** + * Experimental HBase store. + *

+ * This is not ready for production. It's pretty slow. + *

+ * Here are some areas that might need work: + *

+ * - batching? (consider HTable#batch, HTable#setAutoFlush(false) + * - tuning HTable#setWriteBufferSize (?) + * - writing a server-side filter to replace ColumnCountGetFilter, which drops + * all columns on the row where it reaches its limit. This requires getSlice, + * currently, to impose its limit on the client side. That obviously won't + * scale. + * - RowMutations for combining Puts+Deletes (need a newer HBase than 0.92 for this) + * - (maybe) fiddle with HTable#setRegionCachePrefetch and/or #prewarmRegionCache + *

+ * There may be other problem areas. These are just the ones of which I'm aware. + */ +public class HBaseKeyColumnValueStore implements KeyColumnValueStore { + + private static final Logger logger = LoggerFactory.getLogger(HBaseKeyColumnValueStore.class); + + private final String tableName; + private final HTablePool pool; + + private final String columnFamily; + // This is columnFamily.getBytes() + private final byte[] columnFamilyBytes; + + HBaseKeyColumnValueStore(HTablePool pool, String tableName, String columnFamily) { + this.tableName = tableName; + this.pool = pool; + this.columnFamily = columnFamily; + this.columnFamilyBytes = columnFamily.getBytes(); + } + + @Override + public void close() throws StorageException { + try { + pool.close(); + } catch (IOException e) { + throw new TemporaryStorageException(e); + } + } + + @Override + public boolean containsKey(StaticBuffer key, StoreTransaction txh) throws StorageException { + byte[] keyBytes = key.as(StaticBuffer.ARRAY_FACTORY); + + Get g = new Get(keyBytes).addFamily(columnFamilyBytes); + + try { + HTableInterface table = null; + + try { + table = pool.getTable(tableName); + return table.exists(g); + } finally { + IOUtils.closeQuietly(table); + } + } catch (IOException e) { + throw new TemporaryStorageException(e); + } + } + + @Override + public List getSlice(KeySliceQuery query, StoreTransaction txh) throws StorageException { + return getHelper(query.getKey(), getFilter(query)); + } + + public static Filter getFilter(SliceQuery query) { + byte[] colStartBytes = query.getSliceEnd().length()>0 ? query.getSliceStart().as(StaticBuffer.ARRAY_FACTORY) : null; + byte[] colEndBytes = query.getSliceEnd().length()>0 ? query.getSliceEnd().as(StaticBuffer.ARRAY_FACTORY) : null; + + Filter filter = new ColumnRangeFilter(colStartBytes, true, colEndBytes, false); + + if (query.hasLimit()) { + filter = new FilterList(FilterList.Operator.MUST_PASS_ALL, + filter, + new ColumnPaginationFilter(query.getLimit(), 0)); + } + + return filter; + } + + private List getHelper(StaticBuffer key, Filter getFilter) throws StorageException { + byte[] keyBytes = key.as(StaticBuffer.ARRAY_FACTORY); + + Get g = new Get(keyBytes).addFamily(columnFamilyBytes).setFilter(getFilter); + + List ret; + + try { + HTableInterface table = null; + Result r = null; + + try { + table = pool.getTable(tableName); + r = table.get(g); + } finally { + IOUtils.closeQuietly(table); + } + + if (r == null) + return Collections.emptyList(); + + ret = new ArrayList(r.size()); + + Map fmap = r.getFamilyMap(columnFamilyBytes); + + if (null != fmap) { + for (Map.Entry ent : fmap.entrySet()) { + ret.add(StaticBufferEntry.of(new StaticArrayBuffer(ent.getKey()), new StaticArrayBuffer(ent.getValue()))); + } + } + + return ret; + } catch (IOException e) { + throw new TemporaryStorageException(e); + } + } + + @Override + public void mutate(StaticBuffer key, + List additions, + List deletions, + StoreTransaction txh) throws StorageException { + // TODO: use RowMutations (requires 0.94.x-ish HBase), error handling through the legacy batch() method sucks + List batch = makeBatch(columnFamilyBytes, key.as(StaticBuffer.ARRAY_FACTORY), additions, deletions); + + if (batch.isEmpty()) + return; // nothing to apply + + try { + HTableInterface table = null; + + try { + table = pool.getTable(tableName); + table.batch(batch); + table.flushCommits(); + } finally { + IOUtils.closeQuietly(table); + } + } catch (IOException e) { + throw new TemporaryStorageException(e); + } catch (InterruptedException e) { + throw new TemporaryStorageException(e); + } + } + + @Override + public void acquireLock(StaticBuffer key, + StaticBuffer column, + StaticBuffer expectedValue, + StoreTransaction txh) throws StorageException { + throw new UnsupportedOperationException(); + } + + /** + * + * IMPORTANT: Makes the assumption that all keys are 8 byte longs + * + * @param txh + * @return + * @throws StorageException + */ + @Override + public RecordIterator getKeys(StoreTransaction txh) throws StorageException { + Scan s = new Scan().addFamily(columnFamilyBytes); + FilterList fl = new FilterList(); + // returns first instance of a row, then skip to next row + fl.addFilter(new FirstKeyOnlyFilter()); + // only return the Key, don't return the value + fl.addFilter(new KeyOnlyFilter()); + s.setFilter(fl); + + final ResultScanner scanner; + + try { + scanner = pool.getTable(tableName).getScanner(s); + } catch (IOException e) { + throw new PermanentStorageException(e); + } + + return new RecordIterator() { + /* we need to check if key is long serializable because HBase returns weird rows sometimes */ + private final Iterator results = Iterators.filter(scanner.iterator(), new Predicate() { + @Override + public boolean apply(@Nullable Result result) { + if (result == null) + return false; + + try { + StaticBuffer id = new StaticArrayBuffer(result.getRow()); + id.getLong(0); + } catch (NumberFormatException e) { + return false; + } + + return true; + } + }); + + @Override + public boolean hasNext() throws StorageException { + return results.hasNext(); + } + + @Override + public StaticBuffer next() throws StorageException { + return new StaticArrayBuffer(results.next().getRow()); + } + + @Override + public void close() throws StorageException { + scanner.close(); + } + }; + } + + @Override + public StaticBuffer[] getLocalKeyPartition() throws StorageException { + throw new UnsupportedOperationException(); + } + + @Override + public String getName() { + return columnFamily; + } + + /** + * Convert deletions to a Delete command. + * + * @param cfName The name of the ColumnFamily deletions belong to + * @param key The row key + * @param deletions The name of the columns to delete (a.k.a deletions) + * + * @return Delete command or null if deletions were null or empty. + */ + private final static Delete makeDeletionCommand(byte[] cfName, byte[] key, List deletions) { + Preconditions.checkArgument(!deletions.isEmpty()); + + Delete deleteCommand = new Delete(key); + for (StaticBuffer del : deletions) { + deleteCommand.deleteColumn(cfName, del.as(StaticBuffer.ARRAY_FACTORY)); + } + return deleteCommand; + } + + /** + * Convert modification entries into Put command. + * + * @param cfName The name of the ColumnFamily modifications belong to + * @param key The row key + * @param modifications The entries to insert/update. + * + * @return Put command or null if additions were null or empty. + */ + private final static Put makePutCommand(byte[] cfName, byte[] key, List modifications) { + Preconditions.checkArgument(!modifications.isEmpty()); + + Put putCommand = new Put(key); + for (Entry e : modifications) { + putCommand.add(cfName, e.getArrayColumn(), e.getArrayValue()); + } + return putCommand; + } + + public final static List makeBatch(byte[] cfName, byte[] key, List additions, List deletions) { + if (additions.isEmpty() && deletions.isEmpty()) return Collections.emptyList(); + + List batch = new ArrayList(2); + + if (!additions.isEmpty()) { + Put putCommand = makePutCommand(cfName, key, additions); + batch.add(putCommand); + } + + if (!deletions.isEmpty()) { + Delete deleteCommand = makeDeletionCommand(cfName, key, deletions); + batch.add(deleteCommand); + } + return batch; + } +} diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseStoreManager.java b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseStoreManager.java new file mode 100644 index 0000000000..beb88c00f4 --- /dev/null +++ b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseStoreManager.java @@ -0,0 +1,399 @@ +package com.thinkaurelius.titan.diskstorage.hbase; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; +import com.thinkaurelius.titan.core.TitanException; +import com.thinkaurelius.titan.diskstorage.PermanentStorageException; +import com.thinkaurelius.titan.diskstorage.StaticBuffer; +import com.thinkaurelius.titan.diskstorage.StorageException; +import com.thinkaurelius.titan.diskstorage.TemporaryStorageException; +import com.thinkaurelius.titan.diskstorage.common.DistributedStoreManager; +import com.thinkaurelius.titan.diskstorage.keycolumnvalue.*; +import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration; +import com.thinkaurelius.titan.util.system.IOUtils; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HColumnDescriptor; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableNotFoundException; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.io.hfile.Compression; +import org.apache.hadoop.hbase.util.Pair; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * Storage Manager for HBase + * + * @author Dan LaRocque + */ +public class HBaseStoreManager extends DistributedStoreManager implements KeyColumnValueStoreManager { + + private static final Logger logger = LoggerFactory.getLogger(HBaseStoreManager.class); + + public static final String TABLE_NAME_KEY = "tablename"; + public static final String TABLE_NAME_DEFAULT = "titan"; + + public static final int PORT_DEFAULT = 9160; + + public static final String HBASE_CONFIGURATION_NAMESPACE = "hbase-config"; + + public static final ImmutableMap HBASE_CONFIGURATION; + + static { + HBASE_CONFIGURATION = new ImmutableMap.Builder() + .put(GraphDatabaseConfiguration.HOSTNAME_KEY, "hbase.zookeeper.quorum") + .put(GraphDatabaseConfiguration.PORT_KEY, "hbase.zookeeper.property.clientPort") + .build(); + } + + private final String tableName; + private final org.apache.hadoop.conf.Configuration hconf; + + private final ConcurrentMap openStores; + private final HTablePool connectionPool; + + private final StoreFeatures features; + + public HBaseStoreManager(org.apache.commons.configuration.Configuration config) throws StorageException { + super(config, PORT_DEFAULT); + + this.tableName = config.getString(TABLE_NAME_KEY, TABLE_NAME_DEFAULT); + + this.hconf = HBaseConfiguration.create(); + for (Map.Entry confEntry : HBASE_CONFIGURATION.entrySet()) { + if (config.containsKey(confEntry.getKey())) { + hconf.set(confEntry.getValue(), config.getString(confEntry.getKey())); + } + } + + // Copy a subset of our commons config into a Hadoop config + org.apache.commons.configuration.Configuration hbCommons = config.subset(HBASE_CONFIGURATION_NAMESPACE); + + @SuppressWarnings("unchecked") // I hope commons-config eventually fixes this + Iterator keys = hbCommons.getKeys(); + int keysLoaded = 0; + + while (keys.hasNext()) { + String key = keys.next(); + String value = hbCommons.getString(key); + logger.debug("HBase configuration: setting {}={}", key, value); + hconf.set(key, value); + keysLoaded++; + } + + logger.debug("HBase configuration: set a total of {} configuration values", keysLoaded); + + connectionPool = new HTablePool(hconf, connectionPoolSize); + + openStores = new ConcurrentHashMap(); + + // TODO: allowing publicly mutate fields is bad, should be fixed + features = new StoreFeatures(); + features.supportsScan = true; + features.supportsBatchMutation = true; + features.supportsTransactions = false; + features.supportsConsistentKeyOperations = true; + features.supportsLocking = false; + features.isKeyOrdered = false; + features.isDistributed = true; + features.hasLocalKeyPartition = false; + } + + + @Override + public String toString() { + return "hbase[" + tableName + "@" + super.toString() + "]"; + } + + @Override + public void close() { + openStores.clear(); + } + + + @Override + public StoreFeatures getFeatures() { + return features; + } + + @Override + public void mutateMany(Map> mutations, StoreTransaction txh) throws StorageException { + final long delTS = System.currentTimeMillis(); + final long putTS = delTS + 1; + + Map> commandsPerKey = convertToCommands(mutations, putTS, delTS); + List batch = new ArrayList(commandsPerKey.size()); // actual batch operation + + // convert sorted commands into representation required for 'batch' operation + for (Pair commands : commandsPerKey.values()) { + if (commands.getFirst() != null) + batch.add(commands.getFirst()); + + if (commands.getSecond() != null) + batch.add(commands.getSecond()); + } + + try { + HTableInterface table = null; + + try { + table = connectionPool.getTable(tableName); + table.batch(batch); + table.flushCommits(); + } finally { + IOUtils.closeQuietly(table); + } + } catch (IOException e) { + throw new TemporaryStorageException(e); + } catch (InterruptedException e) { + throw new TemporaryStorageException(e); + } + + waitUntil(putTS); + } + + @Override + public KeyColumnValueStore openDatabase(String dbName) throws StorageException { + HBaseKeyColumnValueStore store = openStores.get(dbName); + + if (store == null) { + HBaseKeyColumnValueStore newStore = new HBaseKeyColumnValueStore(connectionPool, tableName, dbName); + + store = openStores.putIfAbsent(dbName, newStore); // nothing bad happens if we loose to other thread + + if (store == null) { // ensure that CF exists only first time somebody tries to open it + ensureColumnFamilyExists(tableName, dbName); + store = newStore; + } + } + + return store; + } + + private HTableDescriptor ensureTableExists(String tableName) throws StorageException { + HBaseAdmin adm = getAdminInterface(); + + HTableDescriptor desc; + + try { // Create our table, if necessary + if (adm.tableExists(tableName)) { + desc = adm.getTableDescriptor(tableName.getBytes()); + } else { + desc = new HTableDescriptor(tableName); + adm.createTable(desc); + } + } catch (IOException e) { + throw new TemporaryStorageException(e); + } + + return desc; + } + + private void ensureColumnFamilyExists(String tableName, String columnFamily) throws StorageException { + HBaseAdmin adm = getAdminInterface(); + HTableDescriptor desc = ensureTableExists(tableName); + + Preconditions.checkNotNull(desc); + + HColumnDescriptor cf = desc.getFamily(columnFamily.getBytes()); + + // Create our column family, if necessary + if (cf == null) { + try { + adm.disableTable(tableName); + desc.addFamily(new HColumnDescriptor(columnFamily).setCompressionType(Compression.Algorithm.GZ)); + adm.modifyTable(tableName.getBytes(), desc); + + try { + logger.debug("Added HBase ColumnFamily {}, waiting for 1 sec. to propogate.", columnFamily); + Thread.sleep(1000L); + } catch (InterruptedException ie) { + throw new TemporaryStorageException(ie); + } + + adm.enableTable(tableName); + } catch (TableNotFoundException ee) { + logger.error("TableNotFoundException", ee); + throw new PermanentStorageException(ee); + } catch (org.apache.hadoop.hbase.TableExistsException ee) { + logger.debug("Swallowing exception {}", ee); + } catch (IOException ee) { + throw new TemporaryStorageException(ee); + } + } else { // check if compression was enabled, if not - enable it + if (cf.getCompressionType() == null || cf.getCompressionType() == Compression.Algorithm.NONE) { + try { + adm.disableTable(tableName); + + adm.modifyColumn(tableName, cf.setCompressionType(Compression.Algorithm.GZ)); + + adm.enableTable(tableName); + } catch (IOException e) { + throw new TemporaryStorageException(e); + } + } + } + } + + @Override + public StoreTransaction beginTransaction(ConsistencyLevel level) throws StorageException { + return new HBaseTransaction(level); + } + + + /** + * Deletes the specified table with all its columns. + * ATTENTION: Invoking this method will delete the table if it exists and therefore causes data loss. + */ + @Override + public void clearStorage() throws StorageException { + HBaseAdmin adm = getAdminInterface(); + + try { // first of all, check if table exists, if not - we are done + if (!adm.tableExists(tableName)) { + logger.debug("clearStorage() called before table {} was created, skipping.", tableName); + return; + } + } catch (IOException e) { + throw new TemporaryStorageException(e); + } + + HTable table = null; + + try { + table = new HTable(hconf, tableName); + + Scan scan = new Scan(); + scan.setBatch(100); + scan.setCacheBlocks(false); + scan.setCaching(2000); + + ResultScanner scanner = null; + + try { + scanner = table.getScanner(scan); + + for (Result res : scanner) { + table.delete(new Delete(res.getRow())); + } + } finally { + IOUtils.closeQuietly(scanner); + } + } catch (IOException e) { + throw new TemporaryStorageException(e); + } finally { + IOUtils.closeQuietly(table); + } + } + + @Override + public String getConfigurationProperty(final String key) throws StorageException { + ensureTableExists(tableName); + + try { + return getAdminInterface().getTableDescriptor(tableName.getBytes()).getValue(key); + } catch (IOException e) { + throw new PermanentStorageException(e); + } + } + + @Override + public void setConfigurationProperty(final String key, final String value) throws StorageException { + byte[] name = tableName.getBytes(); + + HTableDescriptor desc = ensureTableExists(tableName); + + try { + HBaseAdmin adm = getAdminInterface(); + + adm.disableTable(tableName); + + desc.setValue(key, value); + + adm.modifyTable(name, desc); + adm.enableTable(tableName); + } catch (IOException e) { + throw new PermanentStorageException(e); + } + } + + private HBaseAdmin getAdminInterface() { + try { + return new HBaseAdmin(hconf); + } catch (IOException e) { + throw new TitanException(e); + } + } + + /** + * Convert Titan internal Mutation representation into HBase native commands. + * + * @param mutations Mutations to convert into HBase commands. + * @param putTimestamp The timestamp to use for Put commands. + * @param delTimestamp The timestamp to use for Delete commands. + * + * @return Commands sorted by key converted from Titan internal representation. + */ + private static Map> convertToCommands(Map> mutations, + final long putTimestamp, + final long delTimestamp) { + Map> commandsPerKey = new HashMap>(); + + for (Map.Entry> entry : mutations.entrySet()) { + byte[] cfName = entry.getKey().getBytes(); + + for (Map.Entry m : entry.getValue().entrySet()) { + StaticBuffer key = m.getKey(); + KCVMutation mutation = m.getValue(); + + Pair commands = commandsPerKey.get(key); + + if (commands == null) { + commands = new Pair(); + commandsPerKey.put(key, commands); + } + + if (mutation.hasDeletions()) { + if (commands.getSecond() == null) + commands.setSecond(new Delete(key.as(StaticBuffer.ARRAY_FACTORY), delTimestamp, null)); + + for (StaticBuffer b : mutation.getDeletions()) { + commands.getSecond().deleteColumns(cfName, b.as(StaticBuffer.ARRAY_FACTORY), delTimestamp); + } + } + + if (mutation.hasAdditions()) { + if (commands.getFirst() == null) + commands.setFirst(new Put(key.as(StaticBuffer.ARRAY_FACTORY), putTimestamp)); + + for (Entry e : mutation.getAdditions()) { + commands.getFirst().add(cfName, + e.getArrayColumn(), + putTimestamp, + e.getArrayValue()); + } + } + } + } + + return commandsPerKey; + } + + private static void waitUntil(long until) { + long now = System.currentTimeMillis(); + + while (now <= until) { + try { + Thread.sleep(1L); + now = System.currentTimeMillis(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + } +} diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseTransaction.java b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseTransaction.java new file mode 100644 index 0000000000..502f9ba6f1 --- /dev/null +++ b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseTransaction.java @@ -0,0 +1,19 @@ +package com.thinkaurelius.titan.diskstorage.hbase; + +import com.thinkaurelius.titan.diskstorage.common.AbstractStoreTransaction; +import com.thinkaurelius.titan.diskstorage.keycolumnvalue.ConsistencyLevel; + +/** + * This class overrides and adds nothing compared with + * {@link com.thinkaurelius.titan.diskstorage.locking.consistentkey.ConsistentKeyLockTransaction}; however, it creates a transaction type specific + * to HBase, which lets us check for user errors like passing a Cassandra + * transaction into a HBase method. + * + * @author Dan LaRocque + */ +public class HBaseTransaction extends AbstractStoreTransaction { + + public HBaseTransaction(ConsistencyLevel level) { + super(ConsistencyLevel.KEY_CONSISTENT); + } +} diff --git a/titan-accumulo/src/main/resources/log4j.properties b/titan-accumulo/src/main/resources/log4j.properties new file mode 100644 index 0000000000..9b72c91828 --- /dev/null +++ b/titan-accumulo/src/main/resources/log4j.properties @@ -0,0 +1,13 @@ +# A1 is set to be a ConsoleAppender. +log4j.appender.A1=org.apache.log4j.ConsoleAppender + +# A1 uses PatternLayout. +log4j.appender.A1.layout=org.apache.log4j.PatternLayout +log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n + +# Set root logger level to the designated level and its only appender to A1. +log4j.rootLogger=INFO, A1 + +log4j.logger.org.apache.cassandra=INFO +log4j.logger.org.apache.hadoop=INFO +log4j.logger.org.apache.zookeeper=INFO diff --git a/titan-accumulo/src/main/resources/titan.properties b/titan-accumulo/src/main/resources/titan.properties new file mode 100644 index 0000000000..02dc2c4ce5 --- /dev/null +++ b/titan-accumulo/src/main/resources/titan.properties @@ -0,0 +1,2 @@ +titan.version=${project.version} +titan.compatible-versions=${titan.compatible.versions} \ No newline at end of file diff --git a/titan-accumulo/src/test/config/hadoop-metrics.properties b/titan-accumulo/src/test/config/hadoop-metrics.properties new file mode 100644 index 0000000000..2060e22df5 --- /dev/null +++ b/titan-accumulo/src/test/config/hadoop-metrics.properties @@ -0,0 +1,70 @@ +# See http://wiki.apache.org/hadoop/GangliaMetrics +# Make sure you know whether you are using ganglia 3.0 or 3.1. +# If 3.1, you will have to patch your hadoop instance with HADOOP-4675 +# And, yes, this file is named hadoop-metrics.properties rather than +# hbase-metrics.properties because we're leveraging the hadoop metrics +# package and hadoop-metrics.properties is an hardcoded-name, at least +# for the moment. +# +# See also http://hadoop.apache.org/hbase/docs/current/metrics.html +# GMETADHOST_IP is the hostname (or) IP address of the server on which the ganglia +# meta daemon (gmetad) service is running + +# Configuration of the "hbase" context for NullContextWithUpdateThread +# NullContextWithUpdateThread is a null context which has a thread calling +# periodically when monitoring is started. This keeps the data sampled +# correctly. +hbase.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread +hbase.period=10 + +# Configuration of the "hbase" context for file +# hbase.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext +# hbase.fileName=/tmp/metrics_hbase.log + +# HBase-specific configuration to reset long-running stats (e.g. compactions) +# If this variable is left out, then the default is no expiration. +hbase.extendedperiod = 3600 + +# Configuration of the "hbase" context for ganglia +# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) +# hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext +# hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 +# hbase.period=10 +# hbase.servers=GMETADHOST_IP:8649 + +# Configuration of the "jvm" context for null +jvm.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread +jvm.period=10 + +# Configuration of the "jvm" context for file +# jvm.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext +# jvm.fileName=/tmp/metrics_jvm.log + +# Configuration of the "jvm" context for ganglia +# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) +# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext +# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 +# jvm.period=10 +# jvm.servers=GMETADHOST_IP:8649 + +# Configuration of the "rpc" context for null +rpc.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread +rpc.period=10 + +# Configuration of the "rpc" context for file +# rpc.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext +# rpc.fileName=/tmp/metrics_rpc.log + +# Configuration of the "rpc" context for ganglia +# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) +# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext +# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 +# rpc.period=10 +# rpc.servers=GMETADHOST_IP:8649 + +# Configuration of the "rest" context for ganglia +# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) +# rest.class=org.apache.hadoop.metrics.ganglia.GangliaContext +# rest.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 +# rest.period=10 +# rest.servers=GMETADHOST_IP:8649 diff --git a/titan-accumulo/src/test/config/hbase-env.sh b/titan-accumulo/src/test/config/hbase-env.sh new file mode 100644 index 0000000000..13e58f8ef5 --- /dev/null +++ b/titan-accumulo/src/test/config/hbase-env.sh @@ -0,0 +1,94 @@ +# +#/** +# * Copyright 2007 The Apache Software Foundation +# * +# * Licensed to the Apache Software Foundation (ASF) under one +# * or more contributor license agreements. See the NOTICE file +# * distributed with this work for additional information +# * regarding copyright ownership. The ASF licenses this file +# * to you under the Apache License, Version 2.0 (the +# * "License"); you may not use this file except in compliance +# * with the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ + +# Set environment variables here. + +# The java implementation to use. Java 1.6 required. +# export JAVA_HOME=/usr/java/jdk1.6.0/ + +# Extra Java CLASSPATH elements. Optional. +export HBASE_CLASSPATH=$HBASE_CLASSPATH:./target/test-lib/* + +# The maximum amount of heap to use, in MB. Default is 1000. +# export HBASE_HEAPSIZE=1000 + +# Extra Java runtime options. +# Below are what we set by default. May only work with SUN JVM. +# For more on why as well as other possible settings, +# see http://wiki.apache.org/hadoop/PerformanceTuning +export HBASE_OPTS="$HBASE_OPTS -XX:+UseConcMarkSweepGC" + +# Uncomment below to enable java garbage collection logging in the .out file. +# export HBASE_OPTS="$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps $HBASE_GC_OPTS" + +# Uncomment below (along with above GC logging) to put GC information in its own logfile (will set HBASE_GC_OPTS) +# export HBASE_USE_GC_LOGFILE=true + + +# Uncomment below if you intend to use the EXPERIMENTAL off heap cache. +# export HBASE_OPTS="$HBASE_OPTS -XX:MaxDirectMemorySize=" +# Set hbase.offheapcache.percentage in hbase-site.xml to a nonzero value. + + +# Uncomment and adjust to enable JMX exporting +# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access. +# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html +# +# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" +# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10101" +# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10102" +# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103" +# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104" + +# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default. +# export HBASE_REGIONSERVERS=${HBASE_HOME}/conf/regionservers + +# File naming hosts on which backup HMaster will run. $HBASE_HOME/conf/backup-masters by default. +# export HBASE_BACKUP_MASTERS=${HBASE_HOME}/conf/backup-masters + +# Extra ssh options. Empty by default. +# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR" + +# Where log files are stored. $HBASE_HOME/logs by default. +# export HBASE_LOG_DIR=${HBASE_HOME}/logs + +# Enable remote JDWP debugging of major HBase processes. Meant for Core Developers +# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8070" +# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8071" +# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8072" +# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8073" + +# A string representing this instance of hbase. $USER by default. +# export HBASE_IDENT_STRING=$USER + +# The scheduling priority for daemon processes. See 'man nice'. +# export HBASE_NICENESS=10 + +# The directory where pid files are stored. /tmp by default. +# export HBASE_PID_DIR=/var/hadoop/pids + +# Seconds to sleep between slave commands. Unset by default. This +# can be useful in large clusters, where, e.g., slave rsyncs can +# otherwise arrive faster than the master can service them. +# export HBASE_SLAVE_SLEEP=0.1 + +# Tell HBase whether it should manage it's own instance of Zookeeper or not. +# export HBASE_MANAGES_ZK=true diff --git a/titan-accumulo/src/test/config/hbase-policy.xml b/titan-accumulo/src/test/config/hbase-policy.xml new file mode 100644 index 0000000000..e45f23c962 --- /dev/null +++ b/titan-accumulo/src/test/config/hbase-policy.xml @@ -0,0 +1,53 @@ + + + + + + + security.client.protocol.acl + * + ACL for HRegionInterface protocol implementations (ie. + clients talking to HRegionServers) + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed. + + + + security.admin.protocol.acl + * + ACL for HMasterInterface protocol implementation (ie. + clients talking to HMaster for admin operations). + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed. + + + + security.masterregion.protocol.acl + * + ACL for HMasterRegionInterface protocol implementations + (for HRegionServers communicating with HMaster) + The ACL is a comma-separated list of user and group names. The user and + group list is separated by a blank. For e.g. "alice,bob users,wheel". + A special value of "*" means all users are allowed. + + diff --git a/titan-accumulo/src/test/config/hbase-site.xml b/titan-accumulo/src/test/config/hbase-site.xml new file mode 100644 index 0000000000..bf23d12c61 --- /dev/null +++ b/titan-accumulo/src/test/config/hbase-site.xml @@ -0,0 +1,38 @@ + + + + + + + hbase.rootdir + ./src/test/titan-hbase-test-data + + + hbase.zookeeper.property.dataDir + ./src/test/titan-zookeeper-test + + + hbase.master + local + + diff --git a/titan-accumulo/src/test/config/log4j.properties b/titan-accumulo/src/test/config/log4j.properties new file mode 100644 index 0000000000..f6c3305178 --- /dev/null +++ b/titan-accumulo/src/test/config/log4j.properties @@ -0,0 +1,74 @@ +# Define some default values that can be overridden by system properties +hbase.root.logger=INFO,console +hbase.security.logger=INFO,console +hbase.log.dir=. +hbase.log.file=hbase.log + +# Define the root logger to the system property "hbase.root.logger". +log4j.rootLogger=${hbase.root.logger} + +# Logging Threshold +log4j.threshold=ALL + +# +# Daily Rolling File Appender +# +log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender +log4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file} + +# Rollver at midnight +log4j.appender.DRFA.DatePattern=.yyyy-MM-dd + +# 30-day backup +#log4j.appender.DRFA.MaxBackupIndex=30 +log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout + +# Pattern format: Date LogLevel LoggerName LogMessage +log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n + +# Debugging Pattern format +#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n + +# +# Security audit appender +# +hbase.security.log.file=SecurityAuth.audit +log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender +log4j.appender.DRFAS.File=${hbase.log.dir}/${hbase.security.log.file} +log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout +log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n +log4j.category.SecurityLogger=${hbase.security.logger} +log4j.additivity.SecurityLogger=false + +# +# Null Appender +# +log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender + +# +# console +# Add "console" to rootlogger above if you want to use this +# +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.target=System.err +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n + +# Custom Logging levels + +log4j.logger.org.apache.zookeeper=INFO +#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG +log4j.logger.org.apache.hadoop.hbase=DEBUG +# Make these two classes INFO-level. Make them DEBUG to see more zk debug. +log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=INFO +log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher=INFO +#log4j.logger.org.apache.hadoop.dfs=DEBUG +# Set this class to log INFO only otherwise its OTT + +# Uncomment this line to enable tracing on _every_ RPC call (this can be a lot of output) +#log4j.logger.org.apache.hadoop.ipc.HBaseServer.trace=DEBUG + +# Uncomment the below if you want to remove logging of client region caching' +# and scan of .META. messages +# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=INFO +# log4j.logger.org.apache.hadoop.hbase.client.MetaScanner=INFO diff --git a/titan-accumulo/src/test/config/regionservers b/titan-accumulo/src/test/config/regionservers new file mode 100644 index 0000000000..7b9ad531d2 --- /dev/null +++ b/titan-accumulo/src/test/config/regionservers @@ -0,0 +1 @@ +127.0.0.1 diff --git a/titan-accumulo/src/test/config/whirr-hbase.properties b/titan-accumulo/src/test/config/whirr-hbase.properties new file mode 100644 index 0000000000..a9ced05fb7 --- /dev/null +++ b/titan-accumulo/src/test/config/whirr-hbase.properties @@ -0,0 +1,36 @@ +# A Whirr Receipe for HBase + Hadoop +# Read the Configuration Guide for more info: +# http://whirr.apache.org/docs/latest/configuration-guide.html + +# Change the cluster name here +whirr.cluster-name=hbase-hadoop + +# Change the number of machines in the cluster here +whirr.instance-templates=1 zookeeper+hadoop-namenode+hadoop-jobtracker+hbase-master,3 hadoop-datanode+hadoop-tasktracker+hbase-regionserver + +# replication level should not be higher than number of data nodes +hbase-site.dfs.replication=2 + +# For EC2 set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables. +whirr.provider=aws-ec2 +whirr.identity=${env:AWS_ACCESS_KEY_ID} +whirr.credential=${env:AWS_SECRET_ACCESS_KEY} + +# The size of the instance to use. See http://aws.amazon.com/ec2/instance-types/ +whirr.hardware-id=m1.large +# Ubuntu 10.04 LTS Lucid. See http://alestic.com/ +whirr.image-id=us-east-1/ami-da0cf8b3 +# If you choose a different location, make sure whirr.image-id is updated too +whirr.location-id=us-east-1 + +# By default use the user system SSH keys. Override them here. +# whirr.private-key-file=${sys:user.home}/.ssh/id_rsa +# whirr.public-key-file=${whirr.private-key-file}.pub + +whirr.hbase.tarball.url=http://archive.apache.org/dist/hbase/hbase-0.94.1/hbase-0.94.1.tar.gz +whirr.hadoop.tarball.url=http://archive.apache.org/dist/hadoop/core/hadoop-1.0.3/hadoop-1.0.3.tar.gz + +# Options for the hbase master & regionserver processes +#hbase-env.HBASE_MASTER_OPTS=-Xms1000m -Xmx1000m -Xmn256m -XX:+UseConcMarkSweepGC -XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -Xloggc:/data/hbase/logs/hbase-master-gc.log +#hbase-env.HBASE_REGIONSERVER_OPTS=-Xms2000m -Xmx2000m -Xmn256m -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=88 -XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -Xloggc:/data/hbase/logs/hbase-regionserver-gc.log + diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/HBaseStorageSetup.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/HBaseStorageSetup.java new file mode 100644 index 0000000000..90a36cd908 --- /dev/null +++ b/titan-accumulo/src/test/java/com/thinkaurelius/titan/HBaseStorageSetup.java @@ -0,0 +1,119 @@ +package com.thinkaurelius.titan; + +import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration; +import org.apache.commons.configuration.BaseConfiguration; +import org.apache.commons.configuration.Configuration; +import org.apache.commons.io.FileUtils; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; + +public class HBaseStorageSetup { + private static Process HBASE = null; + // amount of seconds to wait before assuming that HBase shutdown + private static final int SHUTDOWN_TIMEOUT_SEC = 20; + + // hbase config for testing + private static final String HBASE_CONFIG_DIR = "./src/test/config"; + + // default pid file location + private static final String HBASE_PID_FILE = "/tmp/hbase-" + System.getProperty("user.name") + "-master.pid"; + + static { + try { + System.out.println("Deleteing old test directories (if any)."); + + // please keep in sync with HBASE_CONFIG_DIR/hbase-site.xml, reading HBase XML config is huge pain. + File hbaseRoot = new File("./src/test/titan-hbase-test-data"); + File zookeeperDataDir = new File("./src/test/titan-zookeeper-test"); + + if (hbaseRoot.exists()) + FileUtils.deleteDirectory(hbaseRoot); + + if (zookeeperDataDir.exists()) + FileUtils.deleteDirectory(zookeeperDataDir); + } catch (IOException e) { + System.err.println("Failed to delete old HBase test directories: '" + e.getMessage() + "', ignoring."); + } + + Runtime.getRuntime().addShutdownHook(new Thread() { + public void run() { + System.out.println("All done. Shutting done HBase."); + + try { + HBaseStorageSetup.shutdownHBase(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + }); + } + + public static Configuration getHBaseStorageConfiguration() { + BaseConfiguration config = new BaseConfiguration(); + config.addProperty(GraphDatabaseConfiguration.STORAGE_BACKEND_KEY, "hbase"); + return config; + } + + public static Configuration getHBaseGraphConfiguration() { + BaseConfiguration config = new BaseConfiguration(); + config.subset(GraphDatabaseConfiguration.STORAGE_NAMESPACE).addProperty(GraphDatabaseConfiguration.STORAGE_BACKEND_KEY, "hbase"); + return config; + } + + public static void startHBase() throws IOException { + if (HBASE != null) + return; // already started, nothing to do + + try { + // start HBase instance with environment set + HBASE = Runtime.getRuntime().exec(String.format("./bin/hbase-daemon.sh --config %s start master", HBASE_CONFIG_DIR)); + + try { + HBASE.waitFor(); // wait for script to return + } catch (InterruptedException e) { + e.printStackTrace(); + } + + assert HBASE.exitValue() >= 0; // check if we have started successfully + } catch (IOException e) { + e.printStackTrace(); + } + } + + private static void shutdownHBase() throws IOException { + if (HBASE == null) + return; // HBase hasn't been started yet + + try { + File pid = new File(HBASE_PID_FILE); + + if (pid.exists()) { + RandomAccessFile pidFile = new RandomAccessFile(pid, "r"); + + StringBuilder b = new StringBuilder(); + + while (pidFile.getFilePointer() < (pidFile.length() - 1)) // we don't need newline + b.append((char) pidFile.readByte()); + + Process kill = Runtime.getRuntime().exec("kill -TERM " + b.toString()); + kill.waitFor(); + + pidFile.close(); + pid.delete(); // delete pid file like nothing happened + + return; + } + + // fall back to scripting + Runtime.getRuntime().exec(String.format("./bin/hbase-daemon.sh --config %s stop master", HBASE_CONFIG_DIR)); + + System.out.println("Waiting 20 seconds for HBase to shutdown..."); + + Thread.sleep(SHUTDOWN_TIMEOUT_SEC * 1000); // wait no longer than timeout seconds + } catch (Exception e) { + throw new AssertionError(e); + } + } +} diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/blueprints/HBaseBlueprintsTest.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/blueprints/HBaseBlueprintsTest.java new file mode 100644 index 0000000000..951d0c041a --- /dev/null +++ b/titan-accumulo/src/test/java/com/thinkaurelius/titan/blueprints/HBaseBlueprintsTest.java @@ -0,0 +1,53 @@ +package com.thinkaurelius.titan.blueprints; + +import com.thinkaurelius.titan.HBaseStorageSetup; +import com.thinkaurelius.titan.core.TitanFactory; +import com.thinkaurelius.titan.diskstorage.StorageException; +import com.thinkaurelius.titan.diskstorage.hbase.HBaseStoreManager; +import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration; +import com.tinkerpop.blueprints.Graph; + +import java.io.IOException; +/** + * @author Matthias Broecheler (me@matthiasb.com) + */ + +public class HBaseBlueprintsTest extends TitanBlueprintsTest { + + @Override + public void startUp() { + try { + HBaseStorageSetup.startHBase(); + } catch (IOException e) { + throw new AssertionError(e); + } + } + + @Override + public void shutDown() { + // we don't need to restart on each test because cleanup is in please + } + + @Override + public Graph generateGraph() { + return TitanFactory.open(HBaseStorageSetup.getHBaseGraphConfiguration()); + } + + @Override + public void cleanUp() throws StorageException { + HBaseStoreManager s = new HBaseStoreManager(HBaseStorageSetup + .getHBaseGraphConfiguration() + .subset(GraphDatabaseConfiguration.STORAGE_NAMESPACE)); + s.clearStorage(); + } + + @Override + public boolean supportsMultipleGraphs() { + return false; + } + + @Override + public Graph generateGraph(String s) { + throw new UnsupportedOperationException(); + } +} diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseKeyColumnValueTest.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseKeyColumnValueTest.java new file mode 100644 index 0000000000..4ed23251d3 --- /dev/null +++ b/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseKeyColumnValueTest.java @@ -0,0 +1,28 @@ +package com.thinkaurelius.titan.diskstorage.hbase; + +import com.thinkaurelius.titan.HBaseStorageSetup; +import com.thinkaurelius.titan.diskstorage.KeyColumnValueStoreTest; +import com.thinkaurelius.titan.diskstorage.StorageException; +import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyColumnValueStoreManager; +import org.apache.commons.configuration.Configuration; +import org.junit.BeforeClass; + +import java.io.IOException; + +public class HBaseKeyColumnValueTest extends KeyColumnValueStoreTest { + @BeforeClass + public static void startHBase() throws IOException { + HBaseStorageSetup.startHBase(); + } + + public KeyColumnValueStoreManager openStorageManager() throws StorageException { + return new HBaseStoreManager(getConfig()); + } + + private Configuration getConfig() { + Configuration c = HBaseStorageSetup.getHBaseStorageConfiguration(); + c.setProperty("hbase-config.hbase.zookeeper.quorum", "localhost"); + c.setProperty("hbase-config.hbase.zookeeper.property.clientPort", "2181"); + return c; + } +} diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseLockKeyColumnValueStoreTest.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseLockKeyColumnValueStoreTest.java new file mode 100644 index 0000000000..ee1c409f44 --- /dev/null +++ b/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseLockKeyColumnValueStoreTest.java @@ -0,0 +1,22 @@ +package com.thinkaurelius.titan.diskstorage.hbase; + +import com.thinkaurelius.titan.HBaseStorageSetup; +import com.thinkaurelius.titan.diskstorage.LockKeyColumnValueStoreTest; +import com.thinkaurelius.titan.diskstorage.StorageException; +import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyColumnValueStoreManager; +import org.apache.commons.configuration.Configuration; +import org.junit.BeforeClass; + +import java.io.IOException; + +public class HBaseLockKeyColumnValueStoreTest extends LockKeyColumnValueStoreTest { + @BeforeClass + public static void startHBase() throws IOException { + HBaseStorageSetup.startHBase(); + } + + public KeyColumnValueStoreManager openStorageManager(int idx) throws StorageException { + Configuration sc = HBaseStorageSetup.getHBaseStorageConfiguration(); + return new HBaseStoreManager(sc); + } +} diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseMultiWriteKeyColumnValueStoreTest.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseMultiWriteKeyColumnValueStoreTest.java new file mode 100644 index 0000000000..cde9fa4225 --- /dev/null +++ b/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseMultiWriteKeyColumnValueStoreTest.java @@ -0,0 +1,26 @@ +package com.thinkaurelius.titan.diskstorage.hbase; + +import com.thinkaurelius.titan.HBaseStorageSetup; +import com.thinkaurelius.titan.diskstorage.MultiWriteKeyColumnValueStoreTest; +import com.thinkaurelius.titan.diskstorage.StorageException; +import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyColumnValueStoreManager; +import org.apache.commons.configuration.Configuration; +import org.junit.BeforeClass; + +import java.io.IOException; + +public class HBaseMultiWriteKeyColumnValueStoreTest extends MultiWriteKeyColumnValueStoreTest { + @BeforeClass + public static void startHBase() throws IOException { + HBaseStorageSetup.startHBase(); + } + + public KeyColumnValueStoreManager openStorageManager() throws StorageException { + return new HBaseStoreManager(getConfig()); + } + + private Configuration getConfig() { + Configuration c = HBaseStorageSetup.getHBaseStorageConfiguration(); + return c; + } +} diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/hbase/HBaseGraphConcurrentTest.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/hbase/HBaseGraphConcurrentTest.java new file mode 100644 index 0000000000..c9980c9594 --- /dev/null +++ b/titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/hbase/HBaseGraphConcurrentTest.java @@ -0,0 +1,18 @@ +package com.thinkaurelius.titan.graphdb.hbase; + +import com.thinkaurelius.titan.HBaseStorageSetup; +import com.thinkaurelius.titan.graphdb.TitanGraphConcurrentTest; +import org.junit.BeforeClass; + +import java.io.IOException; + +public class HBaseGraphConcurrentTest extends TitanGraphConcurrentTest { + @BeforeClass + public static void startHBase() throws IOException { + HBaseStorageSetup.startHBase(); + } + + public HBaseGraphConcurrentTest() { + super(HBaseStorageSetup.getHBaseGraphConfiguration()); + } +} diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/hbase/HBaseGraphPerformanceTest.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/hbase/HBaseGraphPerformanceTest.java new file mode 100644 index 0000000000..fc643b46e0 --- /dev/null +++ b/titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/hbase/HBaseGraphPerformanceTest.java @@ -0,0 +1,18 @@ +package com.thinkaurelius.titan.graphdb.hbase; + +import com.thinkaurelius.titan.HBaseStorageSetup; +import com.thinkaurelius.titan.graphdb.TitanGraphPerformanceTest; +import org.junit.BeforeClass; + +import java.io.IOException; + +public class HBaseGraphPerformanceTest extends TitanGraphPerformanceTest { + @BeforeClass + public static void startHBase() throws IOException { + HBaseStorageSetup.startHBase(); + } + + public HBaseGraphPerformanceTest() { + super(HBaseStorageSetup.getHBaseGraphConfiguration(), 0, 1, false); + } +} diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/hbase/HBaseGraphTest.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/hbase/HBaseGraphTest.java new file mode 100644 index 0000000000..008241bb2a --- /dev/null +++ b/titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/hbase/HBaseGraphTest.java @@ -0,0 +1,18 @@ +package com.thinkaurelius.titan.graphdb.hbase; + +import com.thinkaurelius.titan.HBaseStorageSetup; +import com.thinkaurelius.titan.graphdb.TitanGraphTest; +import org.junit.BeforeClass; + +import java.io.IOException; + +public class HBaseGraphTest extends TitanGraphTest { + @BeforeClass + public static void startHBase() throws IOException { + HBaseStorageSetup.startHBase(); + } + + public HBaseGraphTest() { + super(HBaseStorageSetup.getHBaseGraphConfiguration()); + } +} diff --git a/titan-accumulo/src/test/resources/log4j.properties b/titan-accumulo/src/test/resources/log4j.properties new file mode 100644 index 0000000000..9e12d73134 --- /dev/null +++ b/titan-accumulo/src/test/resources/log4j.properties @@ -0,0 +1,14 @@ +# A1 is set to be a FileAppender. +log4j.appender.A1=org.apache.log4j.FileAppender +log4j.appender.A1.File=target/test.log + +# A1 uses PatternLayout. +log4j.appender.A1.layout=org.apache.log4j.PatternLayout +log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n + +# Set root logger level to the designated level and its only appender to A1. +log4j.rootLogger=INFO, A1 + +log4j.logger.org.apache.cassandra=INFO +log4j.logger.org.apache.hadoop=INFO +log4j.logger.org.apache.zookeeper=INFO \ No newline at end of file diff --git a/titan-accumulo/src/test/resources/rexster-fragment.xml b/titan-accumulo/src/test/resources/rexster-fragment.xml new file mode 100644 index 0000000000..19f88f9236 --- /dev/null +++ b/titan-accumulo/src/test/resources/rexster-fragment.xml @@ -0,0 +1,13 @@ + + + false + home + + local + + + + tp:gremlin + + + From 62b472684f71f6f8a36f57767ae915b3868ccc89 Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Wed, 5 Jun 2013 18:32:54 -0400 Subject: [PATCH 02/50] Skeleton accumulo implementation, fails all tests. --- titan-accumulo/pom.xml | 17 +- .../accumulo/AccumuloKeyColumnValueStore.java | 97 +++++ .../accumulo/AccumuloStoreManager.java | 114 +++++ .../AccumuloTransaction.java} | 6 +- .../hbase/HBaseKeyColumnValueStore.java | 307 -------------- .../diskstorage/hbase/HBaseStoreManager.java | 399 ------------------ .../titan/AccumuloStorageSetup.java | 70 +++ .../titan/HBaseStorageSetup.java | 119 ------ ...sTest.java => AccumuloBlueprintsTest.java} | 14 +- .../accumulo/AccumuloKeyColumnValueTest.java | 29 ++ .../AccumuloLockKeyColumnValueStoreTest.java} | 15 +- ...uloMultiWriteKeyColumnValueStoreTest.java} | 16 +- .../hbase/HBaseKeyColumnValueTest.java | 28 -- .../accumulo/AccumuloGraphConcurrentTest.java | 18 + .../AccumuloGraphPerformanceTest.java | 18 + .../graphdb/accumulo/AccumuloGraphTest.java | 18 + .../hbase/HBaseGraphConcurrentTest.java | 18 - .../hbase/HBaseGraphPerformanceTest.java | 18 - .../titan/graphdb/hbase/HBaseGraphTest.java | 18 - 19 files changed, 394 insertions(+), 945 deletions(-) create mode 100644 titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java create mode 100644 titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java rename titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/{hbase/HBaseTransaction.java => accumulo/AccumuloTransaction.java} (76%) delete mode 100644 titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseKeyColumnValueStore.java delete mode 100644 titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseStoreManager.java create mode 100644 titan-accumulo/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java delete mode 100644 titan-accumulo/src/test/java/com/thinkaurelius/titan/HBaseStorageSetup.java rename titan-accumulo/src/test/java/com/thinkaurelius/titan/blueprints/{HBaseBlueprintsTest.java => AccumuloBlueprintsTest.java} (69%) create mode 100644 titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueTest.java rename titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/{hbase/HBaseLockKeyColumnValueStoreTest.java => accumulo/AccumuloLockKeyColumnValueStoreTest.java} (50%) rename titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/{hbase/HBaseMultiWriteKeyColumnValueStoreTest.java => accumulo/AccumuloMultiWriteKeyColumnValueStoreTest.java} (53%) delete mode 100644 titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseKeyColumnValueTest.java create mode 100644 titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/accumulo/AccumuloGraphConcurrentTest.java create mode 100644 titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/accumulo/AccumuloGraphPerformanceTest.java create mode 100644 titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/accumulo/AccumuloGraphTest.java delete mode 100644 titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/hbase/HBaseGraphConcurrentTest.java delete mode 100644 titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/hbase/HBaseGraphPerformanceTest.java delete mode 100644 titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/hbase/HBaseGraphTest.java diff --git a/titan-accumulo/pom.xml b/titan-accumulo/pom.xml index ec64aca992..f18a1d22e4 100644 --- a/titan-accumulo/pom.xml +++ b/titan-accumulo/pom.xml @@ -28,20 +28,9 @@ ${titan.version} - org.apache.hbase - hbase - - 0.94.1 - - - avro - org.apache.avro - - - jruby-complete - org.jruby - - + org.apache.accumulo + accumulo-core + 1.4.3 org.apache.hadoop diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java new file mode 100644 index 0000000000..fa38290afd --- /dev/null +++ b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java @@ -0,0 +1,97 @@ +package com.thinkaurelius.titan.diskstorage.accumulo; + +import com.thinkaurelius.titan.diskstorage.StaticBuffer; +import com.thinkaurelius.titan.diskstorage.StorageException; +import com.thinkaurelius.titan.diskstorage.keycolumnvalue.*; +import com.thinkaurelius.titan.diskstorage.util.RecordIterator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; + +/** + * Experimental Accumulo store. + *

+ * This is not ready for production. It's pretty slow. + *

+ * Here are some areas that might need work: + *

+ * - batching? (consider HTable#batch, HTable#setAutoFlush(false) + * - tuning HTable#setWriteBufferSize (?) + * - writing a server-side filter to replace ColumnCountGetFilter, which drops + * all columns on the row where it reaches its limit. This requires getSlice, + * currently, to impose its limit on the client side. That obviously won't + * scale. + * - RowMutations for combining Puts+Deletes (need a newer HBase than 0.92 for this) + * - (maybe) fiddle with HTable#setRegionCachePrefetch and/or #prewarmRegionCache + *

+ * There may be other problem areas. These are just the ones of which I'm aware. + */ +public class AccumuloKeyColumnValueStore implements KeyColumnValueStore { + + private static final Logger logger = LoggerFactory.getLogger(AccumuloKeyColumnValueStore.class); + + private final String tableName; + + private final String columnFamily; + // This is columnFamily.getBytes() + private final byte[] columnFamilyBytes; + + AccumuloKeyColumnValueStore(String tableName, String columnFamily) { + this.tableName = tableName; + this.columnFamily = columnFamily; + this.columnFamilyBytes = columnFamily.getBytes(); + } + + @Override + public void close() throws StorageException { + } + + @Override + public boolean containsKey(StaticBuffer key, StoreTransaction txh) throws StorageException { + return false; + } + + @Override + public List getSlice(KeySliceQuery query, StoreTransaction txh) throws StorageException { + return Collections.emptyList(); + } + + @Override + public void mutate(StaticBuffer key, + List additions, + List deletions, + StoreTransaction txh) throws StorageException { + } + + @Override + public void acquireLock(StaticBuffer key, + StaticBuffer column, + StaticBuffer expectedValue, + StoreTransaction txh) throws StorageException { + throw new UnsupportedOperationException(); + } + + /** + * + * IMPORTANT: Makes the assumption that all keys are 8 byte longs + * + * @param txh + * @return + * @throws StorageException + */ + @Override + public RecordIterator getKeys(StoreTransaction txh) throws StorageException { + return null; + } + + @Override + public StaticBuffer[] getLocalKeyPartition() throws StorageException { + throw new UnsupportedOperationException(); + } + + @Override + public String getName() { + return columnFamily; + } +} diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java new file mode 100644 index 0000000000..92127da52b --- /dev/null +++ b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java @@ -0,0 +1,114 @@ +package com.thinkaurelius.titan.diskstorage.accumulo; + +import com.google.common.collect.ImmutableMap; +import com.thinkaurelius.titan.diskstorage.StaticBuffer; +import com.thinkaurelius.titan.diskstorage.StorageException; +import com.thinkaurelius.titan.diskstorage.common.DistributedStoreManager; +import com.thinkaurelius.titan.diskstorage.keycolumnvalue.*; +import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * Storage Manager for HBase + * + * @author Dan LaRocque + */ +public class AccumuloStoreManager extends DistributedStoreManager implements KeyColumnValueStoreManager { + + private static final Logger logger = LoggerFactory.getLogger(AccumuloStoreManager.class); + + public static final String TABLE_NAME_KEY = "tablename"; + public static final String TABLE_NAME_DEFAULT = "titan"; + + public static final int PORT_DEFAULT = 9160; + + public static final String ACCUMULO_CONFIGURATION_NAMESPACE = "accumulo-config"; + + public static final ImmutableMap ACCUMULO_CONFIGURATION; + + static { + ACCUMULO_CONFIGURATION = new ImmutableMap.Builder() + .put(GraphDatabaseConfiguration.HOSTNAME_KEY, "accumulo.zookeeper.quorum") + .put(GraphDatabaseConfiguration.PORT_KEY, "accumulo.zookeeper.property.clientPort") + .build(); + } + + private final String tableName; + + private final ConcurrentMap openStores; + + private final StoreFeatures features; + + public AccumuloStoreManager(org.apache.commons.configuration.Configuration config) throws StorageException { + super(config, PORT_DEFAULT); + + this.tableName = config.getString(TABLE_NAME_KEY, TABLE_NAME_DEFAULT); + + openStores = new ConcurrentHashMap(); + + // TODO: allowing publicly mutate fields is bad, should be fixed + features = new StoreFeatures(); + features.supportsScan = true; + features.supportsBatchMutation = true; + features.supportsTransactions = false; + features.supportsConsistentKeyOperations = true; + features.supportsLocking = false; + features.isKeyOrdered = false; + features.isDistributed = true; + features.hasLocalKeyPartition = false; + } + + + @Override + public String toString() { + return "accumulo[" + tableName + "@" + super.toString() + "]"; + } + + @Override + public void close() { + openStores.clear(); + } + + + @Override + public StoreFeatures getFeatures() { + return features; + } + + @Override + public void mutateMany(Map> mutations, StoreTransaction txh) throws StorageException { + } + + @Override + public KeyColumnValueStore openDatabase(String dbName) throws StorageException { + return null; + } + + + @Override + public StoreTransaction beginTransaction(ConsistencyLevel level) throws StorageException { + return new AccumuloTransaction(level); + } + + + /** + * Deletes the specified table with all its columns. + * ATTENTION: Invoking this method will delete the table if it exists and therefore causes data loss. + */ + @Override + public void clearStorage() throws StorageException { + } + + @Override + public String getConfigurationProperty(final String key) throws StorageException { + return null; + } + + @Override + public void setConfigurationProperty(final String key, final String value) throws StorageException { + } +} diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseTransaction.java b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloTransaction.java similarity index 76% rename from titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseTransaction.java rename to titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloTransaction.java index 502f9ba6f1..97184fd8bb 100644 --- a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseTransaction.java +++ b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloTransaction.java @@ -1,4 +1,4 @@ -package com.thinkaurelius.titan.diskstorage.hbase; +package com.thinkaurelius.titan.diskstorage.accumulo; import com.thinkaurelius.titan.diskstorage.common.AbstractStoreTransaction; import com.thinkaurelius.titan.diskstorage.keycolumnvalue.ConsistencyLevel; @@ -11,9 +11,9 @@ * * @author Dan LaRocque */ -public class HBaseTransaction extends AbstractStoreTransaction { +public class AccumuloTransaction extends AbstractStoreTransaction { - public HBaseTransaction(ConsistencyLevel level) { + public AccumuloTransaction(ConsistencyLevel level) { super(ConsistencyLevel.KEY_CONSISTENT); } } diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseKeyColumnValueStore.java b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseKeyColumnValueStore.java deleted file mode 100644 index f2fc9d5bf0..0000000000 --- a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseKeyColumnValueStore.java +++ /dev/null @@ -1,307 +0,0 @@ -package com.thinkaurelius.titan.diskstorage.hbase; - -import com.google.common.base.Preconditions; -import com.google.common.base.Predicate; -import com.google.common.collect.Iterators; -import com.thinkaurelius.titan.diskstorage.PermanentStorageException; -import com.thinkaurelius.titan.diskstorage.StaticBuffer; -import com.thinkaurelius.titan.diskstorage.StorageException; -import com.thinkaurelius.titan.diskstorage.TemporaryStorageException; -import com.thinkaurelius.titan.diskstorage.keycolumnvalue.*; -import com.thinkaurelius.titan.diskstorage.util.RecordIterator; -import com.thinkaurelius.titan.diskstorage.util.StaticArrayBuffer; -import com.thinkaurelius.titan.util.system.IOUtils; -import org.apache.hadoop.hbase.client.*; -import org.apache.hadoop.hbase.filter.*; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.annotation.Nullable; -import java.io.IOException; -import java.util.*; - -/** - * Experimental HBase store. - *

- * This is not ready for production. It's pretty slow. - *

- * Here are some areas that might need work: - *

- * - batching? (consider HTable#batch, HTable#setAutoFlush(false) - * - tuning HTable#setWriteBufferSize (?) - * - writing a server-side filter to replace ColumnCountGetFilter, which drops - * all columns on the row where it reaches its limit. This requires getSlice, - * currently, to impose its limit on the client side. That obviously won't - * scale. - * - RowMutations for combining Puts+Deletes (need a newer HBase than 0.92 for this) - * - (maybe) fiddle with HTable#setRegionCachePrefetch and/or #prewarmRegionCache - *

- * There may be other problem areas. These are just the ones of which I'm aware. - */ -public class HBaseKeyColumnValueStore implements KeyColumnValueStore { - - private static final Logger logger = LoggerFactory.getLogger(HBaseKeyColumnValueStore.class); - - private final String tableName; - private final HTablePool pool; - - private final String columnFamily; - // This is columnFamily.getBytes() - private final byte[] columnFamilyBytes; - - HBaseKeyColumnValueStore(HTablePool pool, String tableName, String columnFamily) { - this.tableName = tableName; - this.pool = pool; - this.columnFamily = columnFamily; - this.columnFamilyBytes = columnFamily.getBytes(); - } - - @Override - public void close() throws StorageException { - try { - pool.close(); - } catch (IOException e) { - throw new TemporaryStorageException(e); - } - } - - @Override - public boolean containsKey(StaticBuffer key, StoreTransaction txh) throws StorageException { - byte[] keyBytes = key.as(StaticBuffer.ARRAY_FACTORY); - - Get g = new Get(keyBytes).addFamily(columnFamilyBytes); - - try { - HTableInterface table = null; - - try { - table = pool.getTable(tableName); - return table.exists(g); - } finally { - IOUtils.closeQuietly(table); - } - } catch (IOException e) { - throw new TemporaryStorageException(e); - } - } - - @Override - public List getSlice(KeySliceQuery query, StoreTransaction txh) throws StorageException { - return getHelper(query.getKey(), getFilter(query)); - } - - public static Filter getFilter(SliceQuery query) { - byte[] colStartBytes = query.getSliceEnd().length()>0 ? query.getSliceStart().as(StaticBuffer.ARRAY_FACTORY) : null; - byte[] colEndBytes = query.getSliceEnd().length()>0 ? query.getSliceEnd().as(StaticBuffer.ARRAY_FACTORY) : null; - - Filter filter = new ColumnRangeFilter(colStartBytes, true, colEndBytes, false); - - if (query.hasLimit()) { - filter = new FilterList(FilterList.Operator.MUST_PASS_ALL, - filter, - new ColumnPaginationFilter(query.getLimit(), 0)); - } - - return filter; - } - - private List getHelper(StaticBuffer key, Filter getFilter) throws StorageException { - byte[] keyBytes = key.as(StaticBuffer.ARRAY_FACTORY); - - Get g = new Get(keyBytes).addFamily(columnFamilyBytes).setFilter(getFilter); - - List ret; - - try { - HTableInterface table = null; - Result r = null; - - try { - table = pool.getTable(tableName); - r = table.get(g); - } finally { - IOUtils.closeQuietly(table); - } - - if (r == null) - return Collections.emptyList(); - - ret = new ArrayList(r.size()); - - Map fmap = r.getFamilyMap(columnFamilyBytes); - - if (null != fmap) { - for (Map.Entry ent : fmap.entrySet()) { - ret.add(StaticBufferEntry.of(new StaticArrayBuffer(ent.getKey()), new StaticArrayBuffer(ent.getValue()))); - } - } - - return ret; - } catch (IOException e) { - throw new TemporaryStorageException(e); - } - } - - @Override - public void mutate(StaticBuffer key, - List additions, - List deletions, - StoreTransaction txh) throws StorageException { - // TODO: use RowMutations (requires 0.94.x-ish HBase), error handling through the legacy batch() method sucks - List batch = makeBatch(columnFamilyBytes, key.as(StaticBuffer.ARRAY_FACTORY), additions, deletions); - - if (batch.isEmpty()) - return; // nothing to apply - - try { - HTableInterface table = null; - - try { - table = pool.getTable(tableName); - table.batch(batch); - table.flushCommits(); - } finally { - IOUtils.closeQuietly(table); - } - } catch (IOException e) { - throw new TemporaryStorageException(e); - } catch (InterruptedException e) { - throw new TemporaryStorageException(e); - } - } - - @Override - public void acquireLock(StaticBuffer key, - StaticBuffer column, - StaticBuffer expectedValue, - StoreTransaction txh) throws StorageException { - throw new UnsupportedOperationException(); - } - - /** - * - * IMPORTANT: Makes the assumption that all keys are 8 byte longs - * - * @param txh - * @return - * @throws StorageException - */ - @Override - public RecordIterator getKeys(StoreTransaction txh) throws StorageException { - Scan s = new Scan().addFamily(columnFamilyBytes); - FilterList fl = new FilterList(); - // returns first instance of a row, then skip to next row - fl.addFilter(new FirstKeyOnlyFilter()); - // only return the Key, don't return the value - fl.addFilter(new KeyOnlyFilter()); - s.setFilter(fl); - - final ResultScanner scanner; - - try { - scanner = pool.getTable(tableName).getScanner(s); - } catch (IOException e) { - throw new PermanentStorageException(e); - } - - return new RecordIterator() { - /* we need to check if key is long serializable because HBase returns weird rows sometimes */ - private final Iterator results = Iterators.filter(scanner.iterator(), new Predicate() { - @Override - public boolean apply(@Nullable Result result) { - if (result == null) - return false; - - try { - StaticBuffer id = new StaticArrayBuffer(result.getRow()); - id.getLong(0); - } catch (NumberFormatException e) { - return false; - } - - return true; - } - }); - - @Override - public boolean hasNext() throws StorageException { - return results.hasNext(); - } - - @Override - public StaticBuffer next() throws StorageException { - return new StaticArrayBuffer(results.next().getRow()); - } - - @Override - public void close() throws StorageException { - scanner.close(); - } - }; - } - - @Override - public StaticBuffer[] getLocalKeyPartition() throws StorageException { - throw new UnsupportedOperationException(); - } - - @Override - public String getName() { - return columnFamily; - } - - /** - * Convert deletions to a Delete command. - * - * @param cfName The name of the ColumnFamily deletions belong to - * @param key The row key - * @param deletions The name of the columns to delete (a.k.a deletions) - * - * @return Delete command or null if deletions were null or empty. - */ - private final static Delete makeDeletionCommand(byte[] cfName, byte[] key, List deletions) { - Preconditions.checkArgument(!deletions.isEmpty()); - - Delete deleteCommand = new Delete(key); - for (StaticBuffer del : deletions) { - deleteCommand.deleteColumn(cfName, del.as(StaticBuffer.ARRAY_FACTORY)); - } - return deleteCommand; - } - - /** - * Convert modification entries into Put command. - * - * @param cfName The name of the ColumnFamily modifications belong to - * @param key The row key - * @param modifications The entries to insert/update. - * - * @return Put command or null if additions were null or empty. - */ - private final static Put makePutCommand(byte[] cfName, byte[] key, List modifications) { - Preconditions.checkArgument(!modifications.isEmpty()); - - Put putCommand = new Put(key); - for (Entry e : modifications) { - putCommand.add(cfName, e.getArrayColumn(), e.getArrayValue()); - } - return putCommand; - } - - public final static List makeBatch(byte[] cfName, byte[] key, List additions, List deletions) { - if (additions.isEmpty() && deletions.isEmpty()) return Collections.emptyList(); - - List batch = new ArrayList(2); - - if (!additions.isEmpty()) { - Put putCommand = makePutCommand(cfName, key, additions); - batch.add(putCommand); - } - - if (!deletions.isEmpty()) { - Delete deleteCommand = makeDeletionCommand(cfName, key, deletions); - batch.add(deleteCommand); - } - return batch; - } -} diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseStoreManager.java b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseStoreManager.java deleted file mode 100644 index beb88c00f4..0000000000 --- a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseStoreManager.java +++ /dev/null @@ -1,399 +0,0 @@ -package com.thinkaurelius.titan.diskstorage.hbase; - -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMap; -import com.thinkaurelius.titan.core.TitanException; -import com.thinkaurelius.titan.diskstorage.PermanentStorageException; -import com.thinkaurelius.titan.diskstorage.StaticBuffer; -import com.thinkaurelius.titan.diskstorage.StorageException; -import com.thinkaurelius.titan.diskstorage.TemporaryStorageException; -import com.thinkaurelius.titan.diskstorage.common.DistributedStoreManager; -import com.thinkaurelius.titan.diskstorage.keycolumnvalue.*; -import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration; -import com.thinkaurelius.titan.util.system.IOUtils; -import org.apache.hadoop.hbase.HBaseConfiguration; -import org.apache.hadoop.hbase.HColumnDescriptor; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.TableNotFoundException; -import org.apache.hadoop.hbase.client.*; -import org.apache.hadoop.hbase.io.hfile.Compression; -import org.apache.hadoop.hbase.util.Pair; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; - -/** - * Storage Manager for HBase - * - * @author Dan LaRocque - */ -public class HBaseStoreManager extends DistributedStoreManager implements KeyColumnValueStoreManager { - - private static final Logger logger = LoggerFactory.getLogger(HBaseStoreManager.class); - - public static final String TABLE_NAME_KEY = "tablename"; - public static final String TABLE_NAME_DEFAULT = "titan"; - - public static final int PORT_DEFAULT = 9160; - - public static final String HBASE_CONFIGURATION_NAMESPACE = "hbase-config"; - - public static final ImmutableMap HBASE_CONFIGURATION; - - static { - HBASE_CONFIGURATION = new ImmutableMap.Builder() - .put(GraphDatabaseConfiguration.HOSTNAME_KEY, "hbase.zookeeper.quorum") - .put(GraphDatabaseConfiguration.PORT_KEY, "hbase.zookeeper.property.clientPort") - .build(); - } - - private final String tableName; - private final org.apache.hadoop.conf.Configuration hconf; - - private final ConcurrentMap openStores; - private final HTablePool connectionPool; - - private final StoreFeatures features; - - public HBaseStoreManager(org.apache.commons.configuration.Configuration config) throws StorageException { - super(config, PORT_DEFAULT); - - this.tableName = config.getString(TABLE_NAME_KEY, TABLE_NAME_DEFAULT); - - this.hconf = HBaseConfiguration.create(); - for (Map.Entry confEntry : HBASE_CONFIGURATION.entrySet()) { - if (config.containsKey(confEntry.getKey())) { - hconf.set(confEntry.getValue(), config.getString(confEntry.getKey())); - } - } - - // Copy a subset of our commons config into a Hadoop config - org.apache.commons.configuration.Configuration hbCommons = config.subset(HBASE_CONFIGURATION_NAMESPACE); - - @SuppressWarnings("unchecked") // I hope commons-config eventually fixes this - Iterator keys = hbCommons.getKeys(); - int keysLoaded = 0; - - while (keys.hasNext()) { - String key = keys.next(); - String value = hbCommons.getString(key); - logger.debug("HBase configuration: setting {}={}", key, value); - hconf.set(key, value); - keysLoaded++; - } - - logger.debug("HBase configuration: set a total of {} configuration values", keysLoaded); - - connectionPool = new HTablePool(hconf, connectionPoolSize); - - openStores = new ConcurrentHashMap(); - - // TODO: allowing publicly mutate fields is bad, should be fixed - features = new StoreFeatures(); - features.supportsScan = true; - features.supportsBatchMutation = true; - features.supportsTransactions = false; - features.supportsConsistentKeyOperations = true; - features.supportsLocking = false; - features.isKeyOrdered = false; - features.isDistributed = true; - features.hasLocalKeyPartition = false; - } - - - @Override - public String toString() { - return "hbase[" + tableName + "@" + super.toString() + "]"; - } - - @Override - public void close() { - openStores.clear(); - } - - - @Override - public StoreFeatures getFeatures() { - return features; - } - - @Override - public void mutateMany(Map> mutations, StoreTransaction txh) throws StorageException { - final long delTS = System.currentTimeMillis(); - final long putTS = delTS + 1; - - Map> commandsPerKey = convertToCommands(mutations, putTS, delTS); - List batch = new ArrayList(commandsPerKey.size()); // actual batch operation - - // convert sorted commands into representation required for 'batch' operation - for (Pair commands : commandsPerKey.values()) { - if (commands.getFirst() != null) - batch.add(commands.getFirst()); - - if (commands.getSecond() != null) - batch.add(commands.getSecond()); - } - - try { - HTableInterface table = null; - - try { - table = connectionPool.getTable(tableName); - table.batch(batch); - table.flushCommits(); - } finally { - IOUtils.closeQuietly(table); - } - } catch (IOException e) { - throw new TemporaryStorageException(e); - } catch (InterruptedException e) { - throw new TemporaryStorageException(e); - } - - waitUntil(putTS); - } - - @Override - public KeyColumnValueStore openDatabase(String dbName) throws StorageException { - HBaseKeyColumnValueStore store = openStores.get(dbName); - - if (store == null) { - HBaseKeyColumnValueStore newStore = new HBaseKeyColumnValueStore(connectionPool, tableName, dbName); - - store = openStores.putIfAbsent(dbName, newStore); // nothing bad happens if we loose to other thread - - if (store == null) { // ensure that CF exists only first time somebody tries to open it - ensureColumnFamilyExists(tableName, dbName); - store = newStore; - } - } - - return store; - } - - private HTableDescriptor ensureTableExists(String tableName) throws StorageException { - HBaseAdmin adm = getAdminInterface(); - - HTableDescriptor desc; - - try { // Create our table, if necessary - if (adm.tableExists(tableName)) { - desc = adm.getTableDescriptor(tableName.getBytes()); - } else { - desc = new HTableDescriptor(tableName); - adm.createTable(desc); - } - } catch (IOException e) { - throw new TemporaryStorageException(e); - } - - return desc; - } - - private void ensureColumnFamilyExists(String tableName, String columnFamily) throws StorageException { - HBaseAdmin adm = getAdminInterface(); - HTableDescriptor desc = ensureTableExists(tableName); - - Preconditions.checkNotNull(desc); - - HColumnDescriptor cf = desc.getFamily(columnFamily.getBytes()); - - // Create our column family, if necessary - if (cf == null) { - try { - adm.disableTable(tableName); - desc.addFamily(new HColumnDescriptor(columnFamily).setCompressionType(Compression.Algorithm.GZ)); - adm.modifyTable(tableName.getBytes(), desc); - - try { - logger.debug("Added HBase ColumnFamily {}, waiting for 1 sec. to propogate.", columnFamily); - Thread.sleep(1000L); - } catch (InterruptedException ie) { - throw new TemporaryStorageException(ie); - } - - adm.enableTable(tableName); - } catch (TableNotFoundException ee) { - logger.error("TableNotFoundException", ee); - throw new PermanentStorageException(ee); - } catch (org.apache.hadoop.hbase.TableExistsException ee) { - logger.debug("Swallowing exception {}", ee); - } catch (IOException ee) { - throw new TemporaryStorageException(ee); - } - } else { // check if compression was enabled, if not - enable it - if (cf.getCompressionType() == null || cf.getCompressionType() == Compression.Algorithm.NONE) { - try { - adm.disableTable(tableName); - - adm.modifyColumn(tableName, cf.setCompressionType(Compression.Algorithm.GZ)); - - adm.enableTable(tableName); - } catch (IOException e) { - throw new TemporaryStorageException(e); - } - } - } - } - - @Override - public StoreTransaction beginTransaction(ConsistencyLevel level) throws StorageException { - return new HBaseTransaction(level); - } - - - /** - * Deletes the specified table with all its columns. - * ATTENTION: Invoking this method will delete the table if it exists and therefore causes data loss. - */ - @Override - public void clearStorage() throws StorageException { - HBaseAdmin adm = getAdminInterface(); - - try { // first of all, check if table exists, if not - we are done - if (!adm.tableExists(tableName)) { - logger.debug("clearStorage() called before table {} was created, skipping.", tableName); - return; - } - } catch (IOException e) { - throw new TemporaryStorageException(e); - } - - HTable table = null; - - try { - table = new HTable(hconf, tableName); - - Scan scan = new Scan(); - scan.setBatch(100); - scan.setCacheBlocks(false); - scan.setCaching(2000); - - ResultScanner scanner = null; - - try { - scanner = table.getScanner(scan); - - for (Result res : scanner) { - table.delete(new Delete(res.getRow())); - } - } finally { - IOUtils.closeQuietly(scanner); - } - } catch (IOException e) { - throw new TemporaryStorageException(e); - } finally { - IOUtils.closeQuietly(table); - } - } - - @Override - public String getConfigurationProperty(final String key) throws StorageException { - ensureTableExists(tableName); - - try { - return getAdminInterface().getTableDescriptor(tableName.getBytes()).getValue(key); - } catch (IOException e) { - throw new PermanentStorageException(e); - } - } - - @Override - public void setConfigurationProperty(final String key, final String value) throws StorageException { - byte[] name = tableName.getBytes(); - - HTableDescriptor desc = ensureTableExists(tableName); - - try { - HBaseAdmin adm = getAdminInterface(); - - adm.disableTable(tableName); - - desc.setValue(key, value); - - adm.modifyTable(name, desc); - adm.enableTable(tableName); - } catch (IOException e) { - throw new PermanentStorageException(e); - } - } - - private HBaseAdmin getAdminInterface() { - try { - return new HBaseAdmin(hconf); - } catch (IOException e) { - throw new TitanException(e); - } - } - - /** - * Convert Titan internal Mutation representation into HBase native commands. - * - * @param mutations Mutations to convert into HBase commands. - * @param putTimestamp The timestamp to use for Put commands. - * @param delTimestamp The timestamp to use for Delete commands. - * - * @return Commands sorted by key converted from Titan internal representation. - */ - private static Map> convertToCommands(Map> mutations, - final long putTimestamp, - final long delTimestamp) { - Map> commandsPerKey = new HashMap>(); - - for (Map.Entry> entry : mutations.entrySet()) { - byte[] cfName = entry.getKey().getBytes(); - - for (Map.Entry m : entry.getValue().entrySet()) { - StaticBuffer key = m.getKey(); - KCVMutation mutation = m.getValue(); - - Pair commands = commandsPerKey.get(key); - - if (commands == null) { - commands = new Pair(); - commandsPerKey.put(key, commands); - } - - if (mutation.hasDeletions()) { - if (commands.getSecond() == null) - commands.setSecond(new Delete(key.as(StaticBuffer.ARRAY_FACTORY), delTimestamp, null)); - - for (StaticBuffer b : mutation.getDeletions()) { - commands.getSecond().deleteColumns(cfName, b.as(StaticBuffer.ARRAY_FACTORY), delTimestamp); - } - } - - if (mutation.hasAdditions()) { - if (commands.getFirst() == null) - commands.setFirst(new Put(key.as(StaticBuffer.ARRAY_FACTORY), putTimestamp)); - - for (Entry e : mutation.getAdditions()) { - commands.getFirst().add(cfName, - e.getArrayColumn(), - putTimestamp, - e.getArrayValue()); - } - } - } - } - - return commandsPerKey; - } - - private static void waitUntil(long until) { - long now = System.currentTimeMillis(); - - while (now <= until) { - try { - Thread.sleep(1L); - now = System.currentTimeMillis(); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - } -} diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java new file mode 100644 index 0000000000..2ffb1dba3b --- /dev/null +++ b/titan-accumulo/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java @@ -0,0 +1,70 @@ +package com.thinkaurelius.titan; + +import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration; +import org.apache.commons.configuration.BaseConfiguration; +import org.apache.commons.configuration.Configuration; +import org.apache.commons.io.FileUtils; + +import java.io.File; +import java.io.IOException; + +public class AccumuloStorageSetup { + private static Process ACCUMULO = null; + // amount of seconds to wait before assuming that HBase shutdown + private static final int SHUTDOWN_TIMEOUT_SEC = 20; + + // hbase config for testing + private static final String ACCUMULO_CONFIG_DIR = "./src/test/config"; + + // default pid file location + private static final String ACCUMULO_PID_FILE = "/tmp/accumulo-" + System.getProperty("user.name") + "-master.pid"; + + static { + try { + System.out.println("Deleteing old test directories (if any)."); + + // please keep in sync with HBASE_CONFIG_DIR/hbase-site.xml, reading HBase XML config is huge pain. + File accumuloRoot = new File("./src/test/titan-accumulo-test-data"); + File zookeeperDataDir = new File("./src/test/titan-zookeeper-test"); + + if (accumuloRoot.exists()) + FileUtils.deleteDirectory(accumuloRoot); + + if (zookeeperDataDir.exists()) + FileUtils.deleteDirectory(zookeeperDataDir); + } catch (IOException e) { + System.err.println("Failed to delete old Accumulo test directories: '" + e.getMessage() + "', ignoring."); + } + + Runtime.getRuntime().addShutdownHook(new Thread() { + public void run() { + System.out.println("All done. Shutting done Accumulo."); + + try { + AccumuloStorageSetup.shutdownAccumulo(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + }); + } + + public static Configuration getAccumuloStorageConfiguration() { + BaseConfiguration config = new BaseConfiguration(); + config.addProperty(GraphDatabaseConfiguration.STORAGE_BACKEND_KEY, "com.thinkaurelius.titan.diskstorage.accumulo.AccumuloStoreManager"); + return config; + } + + public static Configuration getAccumuloGraphConfiguration() { + BaseConfiguration config = new BaseConfiguration(); + config.subset(GraphDatabaseConfiguration.STORAGE_NAMESPACE).addProperty(GraphDatabaseConfiguration.STORAGE_BACKEND_KEY, + "com.thinkaurelius.titan.diskstorage.accumulo.AccumuloStoreManager"); + return config; + } + + public static void startAccumulo() throws IOException { + } + + private static void shutdownAccumulo() throws IOException { + } +} diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/HBaseStorageSetup.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/HBaseStorageSetup.java deleted file mode 100644 index 90a36cd908..0000000000 --- a/titan-accumulo/src/test/java/com/thinkaurelius/titan/HBaseStorageSetup.java +++ /dev/null @@ -1,119 +0,0 @@ -package com.thinkaurelius.titan; - -import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration; -import org.apache.commons.configuration.BaseConfiguration; -import org.apache.commons.configuration.Configuration; -import org.apache.commons.io.FileUtils; - -import java.io.File; -import java.io.IOException; -import java.io.RandomAccessFile; - -public class HBaseStorageSetup { - private static Process HBASE = null; - // amount of seconds to wait before assuming that HBase shutdown - private static final int SHUTDOWN_TIMEOUT_SEC = 20; - - // hbase config for testing - private static final String HBASE_CONFIG_DIR = "./src/test/config"; - - // default pid file location - private static final String HBASE_PID_FILE = "/tmp/hbase-" + System.getProperty("user.name") + "-master.pid"; - - static { - try { - System.out.println("Deleteing old test directories (if any)."); - - // please keep in sync with HBASE_CONFIG_DIR/hbase-site.xml, reading HBase XML config is huge pain. - File hbaseRoot = new File("./src/test/titan-hbase-test-data"); - File zookeeperDataDir = new File("./src/test/titan-zookeeper-test"); - - if (hbaseRoot.exists()) - FileUtils.deleteDirectory(hbaseRoot); - - if (zookeeperDataDir.exists()) - FileUtils.deleteDirectory(zookeeperDataDir); - } catch (IOException e) { - System.err.println("Failed to delete old HBase test directories: '" + e.getMessage() + "', ignoring."); - } - - Runtime.getRuntime().addShutdownHook(new Thread() { - public void run() { - System.out.println("All done. Shutting done HBase."); - - try { - HBaseStorageSetup.shutdownHBase(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - }); - } - - public static Configuration getHBaseStorageConfiguration() { - BaseConfiguration config = new BaseConfiguration(); - config.addProperty(GraphDatabaseConfiguration.STORAGE_BACKEND_KEY, "hbase"); - return config; - } - - public static Configuration getHBaseGraphConfiguration() { - BaseConfiguration config = new BaseConfiguration(); - config.subset(GraphDatabaseConfiguration.STORAGE_NAMESPACE).addProperty(GraphDatabaseConfiguration.STORAGE_BACKEND_KEY, "hbase"); - return config; - } - - public static void startHBase() throws IOException { - if (HBASE != null) - return; // already started, nothing to do - - try { - // start HBase instance with environment set - HBASE = Runtime.getRuntime().exec(String.format("./bin/hbase-daemon.sh --config %s start master", HBASE_CONFIG_DIR)); - - try { - HBASE.waitFor(); // wait for script to return - } catch (InterruptedException e) { - e.printStackTrace(); - } - - assert HBASE.exitValue() >= 0; // check if we have started successfully - } catch (IOException e) { - e.printStackTrace(); - } - } - - private static void shutdownHBase() throws IOException { - if (HBASE == null) - return; // HBase hasn't been started yet - - try { - File pid = new File(HBASE_PID_FILE); - - if (pid.exists()) { - RandomAccessFile pidFile = new RandomAccessFile(pid, "r"); - - StringBuilder b = new StringBuilder(); - - while (pidFile.getFilePointer() < (pidFile.length() - 1)) // we don't need newline - b.append((char) pidFile.readByte()); - - Process kill = Runtime.getRuntime().exec("kill -TERM " + b.toString()); - kill.waitFor(); - - pidFile.close(); - pid.delete(); // delete pid file like nothing happened - - return; - } - - // fall back to scripting - Runtime.getRuntime().exec(String.format("./bin/hbase-daemon.sh --config %s stop master", HBASE_CONFIG_DIR)); - - System.out.println("Waiting 20 seconds for HBase to shutdown..."); - - Thread.sleep(SHUTDOWN_TIMEOUT_SEC * 1000); // wait no longer than timeout seconds - } catch (Exception e) { - throw new AssertionError(e); - } - } -} diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/blueprints/HBaseBlueprintsTest.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/blueprints/AccumuloBlueprintsTest.java similarity index 69% rename from titan-accumulo/src/test/java/com/thinkaurelius/titan/blueprints/HBaseBlueprintsTest.java rename to titan-accumulo/src/test/java/com/thinkaurelius/titan/blueprints/AccumuloBlueprintsTest.java index 951d0c041a..0c04522f75 100644 --- a/titan-accumulo/src/test/java/com/thinkaurelius/titan/blueprints/HBaseBlueprintsTest.java +++ b/titan-accumulo/src/test/java/com/thinkaurelius/titan/blueprints/AccumuloBlueprintsTest.java @@ -1,9 +1,9 @@ package com.thinkaurelius.titan.blueprints; -import com.thinkaurelius.titan.HBaseStorageSetup; +import com.thinkaurelius.titan.AccumuloStorageSetup; import com.thinkaurelius.titan.core.TitanFactory; import com.thinkaurelius.titan.diskstorage.StorageException; -import com.thinkaurelius.titan.diskstorage.hbase.HBaseStoreManager; +import com.thinkaurelius.titan.diskstorage.accumulo.AccumuloStoreManager; import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration; import com.tinkerpop.blueprints.Graph; @@ -12,12 +12,12 @@ * @author Matthias Broecheler (me@matthiasb.com) */ -public class HBaseBlueprintsTest extends TitanBlueprintsTest { +public class AccumuloBlueprintsTest extends TitanBlueprintsTest { @Override public void startUp() { try { - HBaseStorageSetup.startHBase(); + AccumuloStorageSetup.startAccumulo(); } catch (IOException e) { throw new AssertionError(e); } @@ -30,13 +30,13 @@ public void shutDown() { @Override public Graph generateGraph() { - return TitanFactory.open(HBaseStorageSetup.getHBaseGraphConfiguration()); + return TitanFactory.open(AccumuloStorageSetup.getAccumuloGraphConfiguration()); } @Override public void cleanUp() throws StorageException { - HBaseStoreManager s = new HBaseStoreManager(HBaseStorageSetup - .getHBaseGraphConfiguration() + AccumuloStoreManager s = new AccumuloStoreManager(AccumuloStorageSetup + .getAccumuloGraphConfiguration() .subset(GraphDatabaseConfiguration.STORAGE_NAMESPACE)); s.clearStorage(); } diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueTest.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueTest.java new file mode 100644 index 0000000000..71c61f1f99 --- /dev/null +++ b/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueTest.java @@ -0,0 +1,29 @@ +package com.thinkaurelius.titan.diskstorage.accumulo; + +import com.thinkaurelius.titan.AccumuloStorageSetup; +import com.thinkaurelius.titan.diskstorage.KeyColumnValueStoreTest; +import com.thinkaurelius.titan.diskstorage.StorageException; +import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyColumnValueStoreManager; +import org.apache.commons.configuration.Configuration; +import org.junit.BeforeClass; + +import java.io.IOException; + +public class AccumuloKeyColumnValueTest extends KeyColumnValueStoreTest { + @BeforeClass + public static void startAccmulo() throws IOException { + AccumuloStorageSetup.startAccumulo(); + } + + @Override + public KeyColumnValueStoreManager openStorageManager() throws StorageException { + return new AccumuloStoreManager(getConfig()); + } + + private Configuration getConfig() { + Configuration c = AccumuloStorageSetup.getAccumuloStorageConfiguration(); + c.setProperty("accumulo-config.accumulo.zookeeper.quorum", "localhost"); + c.setProperty("accumulo-config.accumulo.zookeeper.property.clientPort", "2181"); + return c; + } +} diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseLockKeyColumnValueStoreTest.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloLockKeyColumnValueStoreTest.java similarity index 50% rename from titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseLockKeyColumnValueStoreTest.java rename to titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloLockKeyColumnValueStoreTest.java index ee1c409f44..55af9af7a6 100644 --- a/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseLockKeyColumnValueStoreTest.java +++ b/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloLockKeyColumnValueStoreTest.java @@ -1,6 +1,6 @@ -package com.thinkaurelius.titan.diskstorage.hbase; +package com.thinkaurelius.titan.diskstorage.accumulo; -import com.thinkaurelius.titan.HBaseStorageSetup; +import com.thinkaurelius.titan.AccumuloStorageSetup; import com.thinkaurelius.titan.diskstorage.LockKeyColumnValueStoreTest; import com.thinkaurelius.titan.diskstorage.StorageException; import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyColumnValueStoreManager; @@ -9,14 +9,15 @@ import java.io.IOException; -public class HBaseLockKeyColumnValueStoreTest extends LockKeyColumnValueStoreTest { +public class AccumuloLockKeyColumnValueStoreTest extends LockKeyColumnValueStoreTest { @BeforeClass - public static void startHBase() throws IOException { - HBaseStorageSetup.startHBase(); + public static void startAccumulo() throws IOException { + AccumuloStorageSetup.startAccumulo(); } + @Override public KeyColumnValueStoreManager openStorageManager(int idx) throws StorageException { - Configuration sc = HBaseStorageSetup.getHBaseStorageConfiguration(); - return new HBaseStoreManager(sc); + Configuration sc = AccumuloStorageSetup.getAccumuloStorageConfiguration(); + return new AccumuloStoreManager(sc); } } diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseMultiWriteKeyColumnValueStoreTest.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloMultiWriteKeyColumnValueStoreTest.java similarity index 53% rename from titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseMultiWriteKeyColumnValueStoreTest.java rename to titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloMultiWriteKeyColumnValueStoreTest.java index cde9fa4225..04a689d6aa 100644 --- a/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseMultiWriteKeyColumnValueStoreTest.java +++ b/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloMultiWriteKeyColumnValueStoreTest.java @@ -1,6 +1,6 @@ -package com.thinkaurelius.titan.diskstorage.hbase; +package com.thinkaurelius.titan.diskstorage.accumulo; -import com.thinkaurelius.titan.HBaseStorageSetup; +import com.thinkaurelius.titan.AccumuloStorageSetup; import com.thinkaurelius.titan.diskstorage.MultiWriteKeyColumnValueStoreTest; import com.thinkaurelius.titan.diskstorage.StorageException; import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyColumnValueStoreManager; @@ -9,18 +9,20 @@ import java.io.IOException; -public class HBaseMultiWriteKeyColumnValueStoreTest extends MultiWriteKeyColumnValueStoreTest { +public class AccumuloMultiWriteKeyColumnValueStoreTest extends MultiWriteKeyColumnValueStoreTest { + @BeforeClass - public static void startHBase() throws IOException { - HBaseStorageSetup.startHBase(); + public static void startAccumulo() throws IOException { + AccumuloStorageSetup.startAccumulo(); } + @Override public KeyColumnValueStoreManager openStorageManager() throws StorageException { - return new HBaseStoreManager(getConfig()); + return new AccumuloStoreManager(getConfig()); } private Configuration getConfig() { - Configuration c = HBaseStorageSetup.getHBaseStorageConfiguration(); + Configuration c = AccumuloStorageSetup.getAccumuloStorageConfiguration(); return c; } } diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseKeyColumnValueTest.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseKeyColumnValueTest.java deleted file mode 100644 index 4ed23251d3..0000000000 --- a/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/hbase/HBaseKeyColumnValueTest.java +++ /dev/null @@ -1,28 +0,0 @@ -package com.thinkaurelius.titan.diskstorage.hbase; - -import com.thinkaurelius.titan.HBaseStorageSetup; -import com.thinkaurelius.titan.diskstorage.KeyColumnValueStoreTest; -import com.thinkaurelius.titan.diskstorage.StorageException; -import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyColumnValueStoreManager; -import org.apache.commons.configuration.Configuration; -import org.junit.BeforeClass; - -import java.io.IOException; - -public class HBaseKeyColumnValueTest extends KeyColumnValueStoreTest { - @BeforeClass - public static void startHBase() throws IOException { - HBaseStorageSetup.startHBase(); - } - - public KeyColumnValueStoreManager openStorageManager() throws StorageException { - return new HBaseStoreManager(getConfig()); - } - - private Configuration getConfig() { - Configuration c = HBaseStorageSetup.getHBaseStorageConfiguration(); - c.setProperty("hbase-config.hbase.zookeeper.quorum", "localhost"); - c.setProperty("hbase-config.hbase.zookeeper.property.clientPort", "2181"); - return c; - } -} diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/accumulo/AccumuloGraphConcurrentTest.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/accumulo/AccumuloGraphConcurrentTest.java new file mode 100644 index 0000000000..7a4f12dab7 --- /dev/null +++ b/titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/accumulo/AccumuloGraphConcurrentTest.java @@ -0,0 +1,18 @@ +package com.thinkaurelius.titan.graphdb.accumulo; + +import com.thinkaurelius.titan.AccumuloStorageSetup; +import com.thinkaurelius.titan.graphdb.TitanGraphConcurrentTest; +import org.junit.BeforeClass; + +import java.io.IOException; + +public class AccumuloGraphConcurrentTest extends TitanGraphConcurrentTest { + @BeforeClass + public static void startAccumulo() throws IOException { + AccumuloStorageSetup.startAccumulo(); + } + + public AccumuloGraphConcurrentTest() { + super(AccumuloStorageSetup.getAccumuloGraphConfiguration()); + } +} diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/accumulo/AccumuloGraphPerformanceTest.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/accumulo/AccumuloGraphPerformanceTest.java new file mode 100644 index 0000000000..88787ecbfd --- /dev/null +++ b/titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/accumulo/AccumuloGraphPerformanceTest.java @@ -0,0 +1,18 @@ +package com.thinkaurelius.titan.graphdb.accumulo; + +import com.thinkaurelius.titan.AccumuloStorageSetup; +import com.thinkaurelius.titan.graphdb.TitanGraphPerformanceTest; +import org.junit.BeforeClass; + +import java.io.IOException; + +public class AccumuloGraphPerformanceTest extends TitanGraphPerformanceTest { + @BeforeClass + public static void startAccumulo() throws IOException { + AccumuloStorageSetup.startAccumulo(); + } + + public AccumuloGraphPerformanceTest() { + super(AccumuloStorageSetup.getAccumuloGraphConfiguration(), 0, 1, false); + } +} diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/accumulo/AccumuloGraphTest.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/accumulo/AccumuloGraphTest.java new file mode 100644 index 0000000000..a344ebc58a --- /dev/null +++ b/titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/accumulo/AccumuloGraphTest.java @@ -0,0 +1,18 @@ +package com.thinkaurelius.titan.graphdb.accumulo; + +import com.thinkaurelius.titan.AccumuloStorageSetup; +import com.thinkaurelius.titan.graphdb.TitanGraphTest; +import org.junit.BeforeClass; + +import java.io.IOException; + +public class AccumuloGraphTest extends TitanGraphTest { + @BeforeClass + public static void startAccumulo() throws IOException { + AccumuloStorageSetup.startAccumulo(); + } + + public AccumuloGraphTest() { + super(AccumuloStorageSetup.getAccumuloGraphConfiguration()); + } +} diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/hbase/HBaseGraphConcurrentTest.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/hbase/HBaseGraphConcurrentTest.java deleted file mode 100644 index c9980c9594..0000000000 --- a/titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/hbase/HBaseGraphConcurrentTest.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.thinkaurelius.titan.graphdb.hbase; - -import com.thinkaurelius.titan.HBaseStorageSetup; -import com.thinkaurelius.titan.graphdb.TitanGraphConcurrentTest; -import org.junit.BeforeClass; - -import java.io.IOException; - -public class HBaseGraphConcurrentTest extends TitanGraphConcurrentTest { - @BeforeClass - public static void startHBase() throws IOException { - HBaseStorageSetup.startHBase(); - } - - public HBaseGraphConcurrentTest() { - super(HBaseStorageSetup.getHBaseGraphConfiguration()); - } -} diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/hbase/HBaseGraphPerformanceTest.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/hbase/HBaseGraphPerformanceTest.java deleted file mode 100644 index fc643b46e0..0000000000 --- a/titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/hbase/HBaseGraphPerformanceTest.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.thinkaurelius.titan.graphdb.hbase; - -import com.thinkaurelius.titan.HBaseStorageSetup; -import com.thinkaurelius.titan.graphdb.TitanGraphPerformanceTest; -import org.junit.BeforeClass; - -import java.io.IOException; - -public class HBaseGraphPerformanceTest extends TitanGraphPerformanceTest { - @BeforeClass - public static void startHBase() throws IOException { - HBaseStorageSetup.startHBase(); - } - - public HBaseGraphPerformanceTest() { - super(HBaseStorageSetup.getHBaseGraphConfiguration(), 0, 1, false); - } -} diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/hbase/HBaseGraphTest.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/hbase/HBaseGraphTest.java deleted file mode 100644 index 008241bb2a..0000000000 --- a/titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/hbase/HBaseGraphTest.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.thinkaurelius.titan.graphdb.hbase; - -import com.thinkaurelius.titan.HBaseStorageSetup; -import com.thinkaurelius.titan.graphdb.TitanGraphTest; -import org.junit.BeforeClass; - -import java.io.IOException; - -public class HBaseGraphTest extends TitanGraphTest { - @BeforeClass - public static void startHBase() throws IOException { - HBaseStorageSetup.startHBase(); - } - - public HBaseGraphTest() { - super(HBaseStorageSetup.getHBaseGraphConfiguration()); - } -} From 14ed3fb75d72274b17bf31bd29673c73b393b4f2 Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Fri, 7 Jun 2013 10:03:52 -0400 Subject: [PATCH 03/50] First version titan accumulo back-end. --- titan-accumulo/pom.xml | 5 + .../accumulo/AccumuloKeyColumnValueStore.java | 270 +++++++++- .../accumulo/AccumuloStoreManager.java | 279 +++++++++- .../diskstorage/accumulo/MutablePair.java | 109 ++++ .../diskstorage/accumulo/TestAccumulo.java | 44 ++ .../accumulo/AccumuloKeyColumnValueTest.java | 1 - ...muloMultiWriteKeyColumnValueStoreTest.java | 1 - .../accumulo/KeyColumnValueStoreTest.java | 502 ++++++++++++++++++ .../MultiWriteKeyColumnValueStoreTest.java | 414 +++++++++++++++ 9 files changed, 1573 insertions(+), 52 deletions(-) create mode 100644 titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/MutablePair.java create mode 100644 titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/TestAccumulo.java create mode 100644 titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/KeyColumnValueStoreTest.java create mode 100644 titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/MultiWriteKeyColumnValueStoreTest.java diff --git a/titan-accumulo/pom.xml b/titan-accumulo/pom.xml index f18a1d22e4..13c87ca0b0 100644 --- a/titan-accumulo/pom.xml +++ b/titan-accumulo/pom.xml @@ -32,6 +32,11 @@ accumulo-core 1.4.3 + + org.apache.zookeeper + zookeeper + 3.4.5 + org.apache.hadoop hadoop-core diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java index fa38290afd..30382a45c4 100644 --- a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java +++ b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java @@ -1,43 +1,73 @@ package com.thinkaurelius.titan.diskstorage.accumulo; +import com.google.common.base.Preconditions; +import com.thinkaurelius.titan.diskstorage.PermanentStorageException; import com.thinkaurelius.titan.diskstorage.StaticBuffer; import com.thinkaurelius.titan.diskstorage.StorageException; -import com.thinkaurelius.titan.diskstorage.keycolumnvalue.*; +import com.thinkaurelius.titan.diskstorage.keycolumnvalue.Entry; +import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyColumnValueStore; +import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeySliceQuery; +import com.thinkaurelius.titan.diskstorage.keycolumnvalue.StaticBufferEntry; +import com.thinkaurelius.titan.diskstorage.keycolumnvalue.StoreTransaction; import com.thinkaurelius.titan.diskstorage.util.RecordIterator; +import com.thinkaurelius.titan.diskstorage.util.StaticArrayBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import org.apache.accumulo.core.client.BatchScanner; +import org.apache.accumulo.core.client.BatchWriter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - -import java.util.*; +import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.IteratorSetting; +import org.apache.accumulo.core.client.MutationsRejectedException; +import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.iterators.FirstEntryInRowIterator; +import org.apache.accumulo.core.iterators.user.RegExFilter; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.hadoop.io.Text; /** * Experimental Accumulo store. *

- * This is not ready for production. It's pretty slow. + * This is not ready for production. It's pretty slow. *

* Here are some areas that might need work: *

- * - batching? (consider HTable#batch, HTable#setAutoFlush(false) - * - tuning HTable#setWriteBufferSize (?) - * - writing a server-side filter to replace ColumnCountGetFilter, which drops - * all columns on the row where it reaches its limit. This requires getSlice, - * currently, to impose its limit on the client side. That obviously won't - * scale. - * - RowMutations for combining Puts+Deletes (need a newer HBase than 0.92 for this) - * - (maybe) fiddle with HTable#setRegionCachePrefetch and/or #prewarmRegionCache + * - batching? (consider HTable#batch, HTable#setAutoFlush(false) - tuning + * HTable#setWriteBufferSize (?) - writing a server-side filter to replace + * ColumnCountGetFilter, which drops all columns on the row where it reaches its + * limit. This requires getSlice, currently, to impose its limit on the client + * side. That obviously won't scale. - RowMutations for combining Puts+Deletes + * (need a newer HBase than 0.92 for this) - (maybe) fiddle with + * HTable#setRegionCachePrefetch and/or #prewarmRegionCache *

- * There may be other problem areas. These are just the ones of which I'm aware. + * There may be other problem areas. These are just the ones of which I'm aware. */ public class AccumuloKeyColumnValueStore implements KeyColumnValueStore { private static final Logger logger = LoggerFactory.getLogger(AccumuloKeyColumnValueStore.class); - + private static final Authorizations DEFAULT_AUTHORIZATIONS = new Authorizations(); + private static final Long DEFAULT_MAX_MEMORY = 50 * 1024 * 1024l; + private static final Long DEFAULT_MAX_LATENCY = 2 * 60 * 1000l; + private static final Long DEFAULT_TIMEOUT = Long.MAX_VALUE; + private static final Integer DEFAULT_MAX_QUERY_THREADS = 3; + private static final Integer DEFAULT_MAX_WRITE_THREADS = 3; + private final Connector connector; private final String tableName; - private final String columnFamily; // This is columnFamily.getBytes() private final byte[] columnFamilyBytes; - AccumuloKeyColumnValueStore(String tableName, String columnFamily) { + AccumuloKeyColumnValueStore(Connector connector, String tableName, String columnFamily) { + this.connector = connector; this.tableName = tableName; this.columnFamily = columnFamily; this.columnFamilyBytes = columnFamily.getBytes(); @@ -49,26 +79,175 @@ public void close() throws StorageException { @Override public boolean containsKey(StaticBuffer key, StoreTransaction txh) throws StorageException { - return false; + byte[] keyBytes = key.as(StaticBuffer.ARRAY_FACTORY); + try { + Scanner scanner = connector.createScanner(tableName, DEFAULT_AUTHORIZATIONS); + + scanner.setRange(new Range(new Text(keyBytes))); + scanner.fetchColumnFamily(new Text(columnFamilyBytes)); + + return scanner.iterator().hasNext(); + + } catch (TableNotFoundException ex) { + logger.error("Should never throw this exception!", ex); + throw new PermanentStorageException(ex); + } } @Override public List getSlice(KeySliceQuery query, StoreTransaction txh) throws StorageException { - return Collections.emptyList(); + Scanner scanner; + + try { + scanner = connector.createScanner(tableName, DEFAULT_AUTHORIZATIONS); + } catch (TableNotFoundException ex) { + logger.error(ex.getMessage(), ex); + throw new PermanentStorageException(ex); + } + + byte[] keyBytes = query.getKey().as(StaticBuffer.ARRAY_FACTORY); + scanner.setRange(new Range(new Text(keyBytes))); + + IteratorSetting columnfamilyIterator = new IteratorSetting(10, "cfIter", RegExFilter.class); + RegExFilter.setRegexs(columnfamilyIterator, null, columnFamily, null, null, true); + scanner.addScanIterator(columnfamilyIterator); + + byte[] startBytes = query.getSliceStart().as(StaticBuffer.ARRAY_FACTORY); + byte[] endBytes = query.getSliceEnd().as(StaticBuffer.ARRAY_FACTORY); + + int count = 0; + List entries = new ArrayList(); + for (Map.Entry entry : scanner) { + byte[] cqBytes = entry.getKey().getColumnQualifier().getBytes(); + + if (count < query.getLimit()) { + if (startBytes.length == 0 || byteCompare(startBytes, cqBytes) <= 0) { + if (endBytes.length == 0 || byteCompare(cqBytes, endBytes) < 0) { + entries.add(StaticBufferEntry.of(new StaticArrayBuffer(cqBytes), + new StaticArrayBuffer(entry.getValue().get()))); + count++; + } + } + } + } + + return entries; + } + + private static int byteCompare(final byte[] left, final byte[] right) { + return byteCompare(left, 0, left.length, right, 0, right.length); + } + + private static int byteCompare(byte[] buffer1, int offset1, int length1, + byte[] buffer2, int offset2, int length2) { + // Bring WritableComparator code local + int end1 = offset1 + length1; + int end2 = offset2 + length2; + for (int i = offset1, j = offset2; i < end1 && j < end2; i++, j++) { + int a = (buffer1[i] & 0xff); + int b = (buffer2[j] & 0xff); + if (a != b) { + return a - b; + } + } + return length1 - length2; } @Override - public void mutate(StaticBuffer key, - List additions, - List deletions, - StoreTransaction txh) throws StorageException { + public void mutate(StaticBuffer key, List additions, List deletions, + StoreTransaction txh) throws StorageException { + + List batch = makeBatch(columnFamilyBytes, key.as(StaticBuffer.ARRAY_FACTORY), additions, deletions); + + if (batch.isEmpty()) { + return; // nothing to apply + } + + try { + BatchWriter writer = connector.createBatchWriter(tableName, + DEFAULT_MAX_MEMORY, DEFAULT_MAX_LATENCY, DEFAULT_MAX_WRITE_THREADS); + try { + writer.addMutations(batch); + writer.flush(); + } catch (MutationsRejectedException ex) { + logger.error(ex.getMessage(), ex); + throw new PermanentStorageException(ex); + } finally { + try { + writer.close(); + } catch (MutationsRejectedException ex) { + logger.warn(ex.getMessage(), ex); + } + } + } catch (TableNotFoundException ex) { + logger.error(ex.getMessage(), ex); + throw new PermanentStorageException(ex); + } + } + + private static List makeBatch(byte[] cfName, byte[] key, List additions, List deletions) { + if (additions.isEmpty() && deletions.isEmpty()) { + return Collections.emptyList(); + } + + List batch = new ArrayList(2); + + if (!additions.isEmpty()) { + Mutation put = makePutMutation(cfName, key, additions); + batch.add(put); + } + + if (!deletions.isEmpty()) { + Mutation delete = makeDeleteMutation(cfName, key, deletions); + batch.add(delete); + } + return batch; + } + + /** + * Convert deletions to a Delete mutation. + * + * @param cfName The name of the ColumnFamily deletions belong to + * @param key The row key + * @param deletions The name of the columns to delete (a.k.a deletions) + * + * @return Delete command or null if deletions were null or empty. + */ + private static Mutation makeDeleteMutation(byte[] cfName, byte[] key, List deletions) { + Preconditions.checkArgument(!deletions.isEmpty()); + + Mutation mutation = new Mutation(new Text(key)); + for (StaticBuffer del : deletions) { + byte[] cqName = del.as(StaticBuffer.ARRAY_FACTORY); + mutation.putDelete(new Text(cfName), new Text(cqName)); + } + return mutation; + } + + /** + * Convert modification entries into Put command. + * + * @param cfName The name of the ColumnFamily modifications belong to + * @param key The row key + * @param modifications The entries to insert/update. + * + * @return Put command or null if additions were null or empty. + */ + private static Mutation makePutMutation(byte[] cfName, byte[] key, List modifications) { + Preconditions.checkArgument(!modifications.isEmpty()); + + Mutation mutation = new Mutation(new Text(key)); + for (Entry entry : modifications) { + byte[] cqName = entry.getArrayColumn(); + byte[] value = entry.getArrayValue(); + mutation.put(new Text(cfName), new Text(cqName), new Value(value)); + } + return mutation; } @Override - public void acquireLock(StaticBuffer key, - StaticBuffer column, - StaticBuffer expectedValue, - StoreTransaction txh) throws StorageException { + public void acquireLock(StaticBuffer key, StaticBuffer column, StaticBuffer expectedValue, + StoreTransaction txh) throws StorageException { throw new UnsupportedOperationException(); } @@ -82,7 +261,44 @@ public void acquireLock(StaticBuffer key, */ @Override public RecordIterator getKeys(StoreTransaction txh) throws StorageException { - return null; + final BatchScanner scanner; + + try { + scanner = connector.createBatchScanner(tableName, + DEFAULT_AUTHORIZATIONS, DEFAULT_MAX_QUERY_THREADS); + } catch (TableNotFoundException ex) { + logger.error(ex.getMessage(), ex); + throw new PermanentStorageException(ex); + } + + scanner.setRanges(Collections.singletonList(new Range())); + + IteratorSetting columnfamilyIterator = new IteratorSetting(10, "cfIter", RegExFilter.class); + RegExFilter.setRegexs(columnfamilyIterator, null, this.columnFamily, null, null, true); + scanner.addScanIterator(columnfamilyIterator); + + IteratorSetting firstKeyOnlyIterator = new IteratorSetting(20, "keyIter", FirstEntryInRowIterator.class); + scanner.addScanIterator(firstKeyOnlyIterator); + + return new RecordIterator() { + private final Iterator> results = scanner.iterator(); + + @Override + public boolean hasNext() throws StorageException { + return results.hasNext(); + } + + @Override + public StaticBuffer next() throws StorageException { + return new StaticArrayBuffer(results.next().getKey().getRow().getBytes()); + } + + @Override + public void close() throws StorageException { + scanner.close(); + } + }; + } @Override diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java index 92127da52b..aa584da089 100644 --- a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java +++ b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java @@ -1,16 +1,41 @@ package com.thinkaurelius.titan.diskstorage.accumulo; import com.google.common.collect.ImmutableMap; +import com.thinkaurelius.titan.diskstorage.PermanentStorageException; import com.thinkaurelius.titan.diskstorage.StaticBuffer; import com.thinkaurelius.titan.diskstorage.StorageException; import com.thinkaurelius.titan.diskstorage.common.DistributedStoreManager; -import com.thinkaurelius.titan.diskstorage.keycolumnvalue.*; +import com.thinkaurelius.titan.diskstorage.keycolumnvalue.ConsistencyLevel; +import com.thinkaurelius.titan.diskstorage.keycolumnvalue.Entry; +import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KCVMutation; +import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyColumnValueStore; +import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyColumnValueStoreManager; +import com.thinkaurelius.titan.diskstorage.keycolumnvalue.StoreFeatures; +import com.thinkaurelius.titan.diskstorage.keycolumnvalue.StoreTransaction; import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.BatchDeleter; +import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.Instance; +import org.apache.accumulo.core.client.MutationsRejectedException; +import org.apache.accumulo.core.client.TableExistsException; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.client.ZooKeeperInstance; +import org.apache.accumulo.core.client.admin.TableOperations; +import org.apache.accumulo.core.client.mock.MockInstance; +import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.commons.configuration.Configuration; +import org.apache.hadoop.io.Text; /** * Storage Manager for HBase @@ -20,36 +45,49 @@ public class AccumuloStoreManager extends DistributedStoreManager implements KeyColumnValueStoreManager { private static final Logger logger = LoggerFactory.getLogger(AccumuloStoreManager.class); - + private static final Authorizations DEFAULT_AUTHORIZATIONS = new Authorizations(); + private static final Long DEFAULT_MAX_MEMORY = 50 * 1024 * 1024l; + private static final Long DEFAULT_MAX_LATENCY = 2 * 60 * 1000l; + private static final Long DEFAULT_TIMEOUT = Long.MAX_VALUE; + private static final Integer DEFAULT_MAX_QUERY_THREADS = 3; + private static final Integer DEFAULT_MAX_WRITE_THREADS = 3; public static final String TABLE_NAME_KEY = "tablename"; public static final String TABLE_NAME_DEFAULT = "titan"; - public static final int PORT_DEFAULT = 9160; - public static final String ACCUMULO_CONFIGURATION_NAMESPACE = "accumulo-config"; - public static final ImmutableMap ACCUMULO_CONFIGURATION; - + static { ACCUMULO_CONFIGURATION = new ImmutableMap.Builder() - .put(GraphDatabaseConfiguration.HOSTNAME_KEY, "accumulo.zookeeper.quorum") - .put(GraphDatabaseConfiguration.PORT_KEY, "accumulo.zookeeper.property.clientPort") - .build(); + .put(GraphDatabaseConfiguration.HOSTNAME_KEY, "accumulo.zookeeper.quorum") + .put(GraphDatabaseConfiguration.PORT_KEY, "accumulo.zookeeper.property.clientPort") + .build(); } - private final String tableName; - private final ConcurrentMap openStores; - + private final Instance instance; + private final Connector connector; private final StoreFeatures features; - public AccumuloStoreManager(org.apache.commons.configuration.Configuration config) throws StorageException { + public AccumuloStoreManager(Configuration config) throws StorageException { super(config, PORT_DEFAULT); this.tableName = config.getString(TABLE_NAME_KEY, TABLE_NAME_DEFAULT); openStores = new ConcurrentHashMap(); + instance = new ZooKeeperInstance("EtCloud", "localhost"); + // instance = new MockInstance("EtCloud"); + try { + connector = instance.getConnector("root", "bobross".getBytes()); + } catch (AccumuloException ex) { + logger.error(ex.getMessage(), ex); + throw new PermanentStorageException(ex.getMessage(), ex); + } catch (AccumuloSecurityException ex) { + logger.error(ex.getMessage(), ex); + throw new PermanentStorageException(ex.getMessage(), ex); + } + // TODO: allowing publicly mutate fields is bad, should be fixed features = new StoreFeatures(); features.supportsScan = true; @@ -62,7 +100,6 @@ public AccumuloStoreManager(org.apache.commons.configuration.Configuration confi features.hasLocalKeyPartition = false; } - @Override public String toString() { return "accumulo[" + tableName + "@" + super.toString() + "]"; @@ -70,45 +107,241 @@ public String toString() { @Override public void close() { - openStores.clear(); } - @Override public StoreFeatures getFeatures() { return features; } @Override - public void mutateMany(Map> mutations, StoreTransaction txh) throws StorageException { + public void mutateMany(Map> mutations, + StoreTransaction txh) throws StorageException { + final long delTS = System.currentTimeMillis(); + final long putTS = delTS + 1; + + Map> commandsPerKey = convertToCommands(mutations, putTS, delTS); + + + List batch = new ArrayList(commandsPerKey.size()); // actual batch operation + // convert sorted commands into representation required for 'batch' operation + for (MutablePair commands : commandsPerKey.values()) { + if (commands.getFirst() != null) { + batch.add(commands.getFirst()); + } + + if (commands.getSecond() != null) { + batch.add(commands.getSecond()); + } + } + + try { + BatchWriter writer = connector.createBatchWriter(tableName, + DEFAULT_MAX_MEMORY, DEFAULT_MAX_LATENCY, DEFAULT_MAX_WRITE_THREADS); + try { + writer.addMutations(batch); + writer.flush(); + } catch (MutationsRejectedException ex) { + logger.error(ex.getMessage(), ex); + throw new PermanentStorageException(ex); + } finally { + if (writer != null) { + try { + writer.close(); + } catch (MutationsRejectedException ex) { + logger.warn(ex.getMessage(), ex); + } + } + } + } catch (TableNotFoundException ex) { + logger.error(ex.getMessage(), ex); + throw new PermanentStorageException(ex); + } + + waitUntil(putTS); + } + + /** + * Convert Titan internal Mutation representation into Accumulo native + * commands. + * + * @param mutations Mutations to convert into Accumulo commands. + * @param putTimestamp The timestamp to use for Put mutations. + * @param delTimestamp The timestamp to use for Delete mutations. + * + * @return Commands sorted by key converted from Titan internal + * representation. + */ + private static Map> convertToCommands(Map> mutations, + final long putTimestamp, final long delTimestamp) { + Map> commandsPerKey = + new HashMap>(); + + for (Map.Entry> entry : mutations.entrySet()) { + Text cfName = new Text(entry.getKey().getBytes()); + + for (Map.Entry m : entry.getValue().entrySet()) { + StaticBuffer key = m.getKey(); + KCVMutation mutation = m.getValue(); + + MutablePair commands = commandsPerKey.get(key); + + if (commands == null) { + commands = new MutablePair(null, null); + commandsPerKey.put(key, commands); + } + + if (mutation.hasDeletions()) { + if (commands.getSecond() == null) { + commands.setSecond(new Mutation(new Text(key.as(StaticBuffer.ARRAY_FACTORY)))); + } + + for (StaticBuffer b : mutation.getDeletions()) { + commands.getSecond().putDelete(cfName, new Text(b.as(StaticBuffer.ARRAY_FACTORY)), delTimestamp); + } + } + + if (mutation.hasAdditions()) { + if (commands.getFirst() == null) { + commands.setFirst(new Mutation(new Text(key.as(StaticBuffer.ARRAY_FACTORY)))); + } + + for (Entry e : mutation.getAdditions()) { + commands.getFirst().put(cfName, new Text(e.getArrayColumn()), + putTimestamp, new Value(e.getArrayValue())); + } + } + } + } + + return commandsPerKey; } @Override public KeyColumnValueStore openDatabase(String dbName) throws StorageException { - return null; + AccumuloKeyColumnValueStore store = openStores.get(dbName); + + if (store == null) { + AccumuloKeyColumnValueStore newStore = new AccumuloKeyColumnValueStore(connector, tableName, dbName); + + store = openStores.putIfAbsent(dbName, newStore); // nothing bad happens if we loose to other thread + + if (store == null) { // ensure that CF exists only first time somebody tries to open it + ensureColumnFamilyExists(tableName, dbName); + store = newStore; + } + } + + return store; + } + + private void ensureTableExists(String tableName) throws StorageException { + TableOperations operations = connector.tableOperations(); + if (!operations.exists(tableName)) { + try { + operations.create(tableName); + } catch (AccumuloException ex) { + logger.error(ex.getMessage(), ex); + throw new PermanentStorageException(ex); + } catch (AccumuloSecurityException ex) { + logger.error(ex.getMessage(), ex); + throw new PermanentStorageException(ex); + } catch (TableExistsException ex) { + logger.warn("Should never throw this exception!", ex); + } + } + } + + private void ensureColumnFamilyExists(String tableName, String columnFamily) throws StorageException { + ensureTableExists(tableName); } - @Override public StoreTransaction beginTransaction(ConsistencyLevel level) throws StorageException { return new AccumuloTransaction(level); } - /** - * Deletes the specified table with all its columns. - * ATTENTION: Invoking this method will delete the table if it exists and therefore causes data loss. + * Deletes the specified table with all its columns. ATTENTION: Invoking + * this method will delete the table if it exists and therefore causes data + * loss. */ @Override public void clearStorage() throws StorageException { + TableOperations operations = connector.tableOperations(); + + // first of all, check if table exists, if not - we are done + if (!operations.exists(tableName)) { + logger.debug("clearStorage() called before table {} was created, skipping.", tableName); + return; + } + try { + BatchDeleter deleter = connector.createBatchDeleter(tableName, DEFAULT_AUTHORIZATIONS, + DEFAULT_MAX_QUERY_THREADS, DEFAULT_MAX_MEMORY, DEFAULT_MAX_LATENCY, DEFAULT_MAX_WRITE_THREADS); + deleter.setRanges(Collections.singletonList(new Range())); + try { + deleter.delete(); + } catch (MutationsRejectedException ex) { + logger.error(ex.getMessage(), ex); + throw new PermanentStorageException(ex); + } finally { + deleter.close(); + } + + } catch (TableNotFoundException ex) { + logger.error(ex.getMessage(), ex); + throw new PermanentStorageException(ex); + } } @Override public String getConfigurationProperty(final String key) throws StorageException { - return null; + ensureTableExists(tableName); + + TableOperations operations = connector.tableOperations(); + try { + Iterable> propIterator = operations.getProperties(tableName); + + Map properties = new HashMap(); + for (Map.Entry entry : propIterator) { + properties.put(entry.getKey(), entry.getValue()); + } + return properties.get(key); + } catch (AccumuloException ex) { + logger.error(ex.getMessage(), ex); + throw new PermanentStorageException(ex); + } catch (TableNotFoundException ex) { + logger.error(ex.getMessage(), ex); + throw new PermanentStorageException(ex); + } } @Override public void setConfigurationProperty(final String key, final String value) throws StorageException { + ensureTableExists(tableName); + + TableOperations operations = connector.tableOperations(); + try { + operations.setProperty(tableName, key, value); + } catch (AccumuloException ex) { + logger.error(ex.getMessage(), ex); + throw new PermanentStorageException(ex); + } catch (AccumuloSecurityException ex) { + logger.error(ex.getMessage(), ex); + throw new PermanentStorageException(ex); + } + } + + private static void waitUntil(long until) { + long now = System.currentTimeMillis(); + + while (now <= until) { + try { + Thread.sleep(1L); + now = System.currentTimeMillis(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } } } diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/MutablePair.java b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/MutablePair.java new file mode 100644 index 0000000000..747b0422d1 --- /dev/null +++ b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/MutablePair.java @@ -0,0 +1,109 @@ +/* + * To change this template, choose Tools | Templates + * and open the template in the editor. + */ +package com.thinkaurelius.titan.diskstorage.accumulo; + +/** + * A generic class for pairs. + * + * @param + * @param + */ +public class MutablePair { + + protected T1 first = null; + protected T2 second = null; + + /** + * Default constructor. + */ + public MutablePair() { + } + + /** + * Constructor + * + * @param a operand + * @param b operand + */ + public MutablePair(T1 a, T2 b) { + this.first = a; + this.second = b; + } + + /** + * Constructs a new pair, inferring the type via the passed arguments + * + * @param type for first + * @param type for second + * @param a first element + * @param b second element + * @return a new pair containing the passed arguments + */ + public static MutablePair newPair(T1 a, T2 b) { + return new MutablePair(a, b); + } + + /** + * Replace the first element of the pair. + * + * @param a operand + */ + public void setFirst(T1 a) { + this.first = a; + } + + /** + * Replace the second element of the pair. + * + * @param b operand + */ + public void setSecond(T2 b) { + this.second = b; + } + + /** + * Return the first element stored in the pair. + * + * @return T1 + */ + public T1 getFirst() { + return first; + } + + /** + * Return the second element stored in the pair. + * + * @return T2 + */ + public T2 getSecond() { + return second; + } + + private static boolean equals(Object x, Object y) { + return (x == null && y == null) || (x != null && x.equals(y)); + } + + @Override + public boolean equals(Object other) { + return other instanceof MutablePair && equals(first, ((MutablePair) other).first) + && equals(second, ((MutablePair) other).second); + } + + @Override + public int hashCode() { + if (first == null) { + return (second == null) ? 0 : second.hashCode() + 1; + } else if (second == null) { + return first.hashCode() + 2; + } else { + return first.hashCode() * 17 + second.hashCode(); + } + } + + @Override + public String toString() { + return "{" + getFirst() + "," + getSecond() + "}"; + } +} \ No newline at end of file diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/TestAccumulo.java b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/TestAccumulo.java new file mode 100644 index 0000000000..6113606f44 --- /dev/null +++ b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/TestAccumulo.java @@ -0,0 +1,44 @@ +/* + * To change this template, choose Tools | Templates + * and open the template in the editor. + */ +package com.thinkaurelius.titan.diskstorage.accumulo; + +import com.thinkaurelius.titan.diskstorage.StaticBuffer; +import com.thinkaurelius.titan.diskstorage.keycolumnvalue.ConsistencyLevel; +import com.thinkaurelius.titan.diskstorage.keycolumnvalue.Entry; +import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyColumnValueStore; +import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyColumnValueStoreManager; +import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeySliceQuery; +import com.thinkaurelius.titan.diskstorage.keycolumnvalue.StaticBufferEntry; +import com.thinkaurelius.titan.diskstorage.keycolumnvalue.StoreTransaction; +import com.thinkaurelius.titan.diskstorage.util.ByteBufferUtil; +import com.thinkaurelius.titan.diskstorage.util.StaticArrayBuffer; +import java.util.Collections; +import org.apache.commons.configuration.BaseConfiguration; + +/** + * + * @author edeprit + */ +public class TestAccumulo { + + public static void main(String[] args) throws Exception { + KeyColumnValueStoreManager manager = new AccumuloStoreManager(new BaseConfiguration()); + KeyColumnValueStore store = manager.openDatabase("foo"); + + StoreTransaction txh = manager.beginTransaction(ConsistencyLevel.DEFAULT); + + Entry entry = new StaticBufferEntry(ByteBufferUtil.getIntBuffer(2), ByteBufferUtil.getIntBuffer(3)); + + StaticBuffer key = ByteBufferUtil.getIntBuffer(1); + + store.mutate(key, Collections.singletonList(entry), + KeyColumnValueStore.NO_DELETIONS, txh); + + KeySliceQuery query = new KeySliceQuery(key, new StaticArrayBuffer(new byte[0]), new StaticArrayBuffer(new byte[0])); + for (Entry e : store.getSlice(query, txh)) { + System.out.println(e); + } + } +} diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueTest.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueTest.java index 71c61f1f99..d2af385137 100644 --- a/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueTest.java +++ b/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueTest.java @@ -1,7 +1,6 @@ package com.thinkaurelius.titan.diskstorage.accumulo; import com.thinkaurelius.titan.AccumuloStorageSetup; -import com.thinkaurelius.titan.diskstorage.KeyColumnValueStoreTest; import com.thinkaurelius.titan.diskstorage.StorageException; import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyColumnValueStoreManager; import org.apache.commons.configuration.Configuration; diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloMultiWriteKeyColumnValueStoreTest.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloMultiWriteKeyColumnValueStoreTest.java index 04a689d6aa..c5f8b1385e 100644 --- a/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloMultiWriteKeyColumnValueStoreTest.java +++ b/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloMultiWriteKeyColumnValueStoreTest.java @@ -1,7 +1,6 @@ package com.thinkaurelius.titan.diskstorage.accumulo; import com.thinkaurelius.titan.AccumuloStorageSetup; -import com.thinkaurelius.titan.diskstorage.MultiWriteKeyColumnValueStoreTest; import com.thinkaurelius.titan.diskstorage.StorageException; import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyColumnValueStoreManager; import org.apache.commons.configuration.Configuration; diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/KeyColumnValueStoreTest.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/KeyColumnValueStoreTest.java new file mode 100644 index 0000000000..18234b104e --- /dev/null +++ b/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/KeyColumnValueStoreTest.java @@ -0,0 +1,502 @@ +package com.thinkaurelius.titan.diskstorage.accumulo; + + +import com.google.common.collect.Sets; +import com.thinkaurelius.titan.diskstorage.KeyColumn; +import com.thinkaurelius.titan.diskstorage.KeyColumnValueStoreUtil; +import com.thinkaurelius.titan.diskstorage.KeyValueStoreUtil; +import com.thinkaurelius.titan.diskstorage.StaticBuffer; +import com.thinkaurelius.titan.diskstorage.StorageException; +import com.thinkaurelius.titan.diskstorage.keycolumnvalue.*; +import com.thinkaurelius.titan.diskstorage.util.RecordIterator; +import com.thinkaurelius.titan.testutil.RandomGenerator; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; + +public abstract class KeyColumnValueStoreTest { + + private Logger log = LoggerFactory.getLogger(KeyColumnValueStoreTest.class); + + int numKeys = 50; + int numColumns = 5; + + protected String storeName = "testStore1"; + + public KeyColumnValueStoreManager manager; + public StoreTransaction tx; + public KeyColumnValueStore store; + + @Before + public void setUp() throws Exception { + openStorageManager().clearStorage(); + open(); + } + + public abstract KeyColumnValueStoreManager openStorageManager() throws StorageException; + + public void open() throws StorageException { + manager = openStorageManager(); + tx = manager.beginTransaction(ConsistencyLevel.DEFAULT); + store = manager.openDatabase(storeName); + } + + public void clopen() throws StorageException { + close(); + open(); + } + + @After + public void tearDown() throws Exception { + close(); + } + + public void close() throws StorageException { + if (tx != null) tx.commit(); + store.close(); + manager.close(); + } + + @Test + public void createDatabase() { + //Just setup and shutdown + } + + + public String[][] generateValues() { + return KeyValueStoreUtil.generateData(numKeys, numColumns); + } + + public void loadValues(String[][] values) throws StorageException { + for (int i = 0; i < values.length; i++) { + List entries = new ArrayList(); + for (int j = 0; j < values[i].length; j++) { + entries.add(new StaticBufferEntry(KeyValueStoreUtil.getBuffer(j), KeyValueStoreUtil.getBuffer(values[i][j]))); + } + store.mutate(KeyValueStoreUtil.getBuffer(i), entries, KeyColumnValueStore.NO_DELETIONS, tx); + } + } + + public Set deleteValues(int every) throws StorageException { + Set removed = new HashSet(); + int counter = 0; + for (int i = 0; i < numKeys; i++) { + List deletions = new ArrayList(); + for (int j = 0; j < numColumns; j++) { + counter++; + if (counter % every == 0) { + //remove + removed.add(new KeyColumn(i, j)); + deletions.add(KeyValueStoreUtil.getBuffer(j)); + } + } + store.mutate(KeyValueStoreUtil.getBuffer(i), KeyColumnValueStore.NO_ADDITIONS, deletions, tx); + } + return removed; + } + + public Set deleteKeys(int every) throws StorageException { + Set removed = new HashSet(); + for (int i = 0; i < numKeys; i++) { + if (i % every == 0) { + removed.add(i); + List deletions = new ArrayList(); + for (int j = 0; j < numColumns; j++) { + deletions.add(KeyValueStoreUtil.getBuffer(j)); + } + store.mutate(KeyValueStoreUtil.getBuffer(i), KeyColumnValueStore.NO_ADDITIONS, deletions, tx); + } + } + return removed; + } + + public void checkKeys(Set removed) throws StorageException { + for (int i = 0; i < numKeys; i++) { + if (removed.contains(i)) { + Assert.assertFalse(store.containsKey(KeyValueStoreUtil.getBuffer(i), tx)); + } else { + Assert.assertTrue(store.containsKey(KeyValueStoreUtil.getBuffer(i), tx)); + } + } + } + + public void checkValueExistence(String[][] values) throws StorageException { + checkValueExistence(values, new HashSet()); + } + + public void checkValueExistence(String[][] values, Set removed) throws StorageException { + for (int i = 0; i < numKeys; i++) { + for (int j = 0; j < numColumns; j++) { + boolean result = KCVSUtil.containsKeyColumn(store, KeyValueStoreUtil.getBuffer(i), KeyValueStoreUtil.getBuffer(j), tx); + if (removed.contains(new KeyColumn(i, j))) { + Assert.assertFalse(result); + } else { + Assert.assertTrue(result); + } + } + } + } + + public void checkValues(String[][] values) throws StorageException { + checkValues(values, new HashSet()); + } + + public void checkValues(String[][] values, Set removed) throws StorageException { + for (int i = 0; i < numKeys; i++) { + for (int j = 0; j < numColumns; j++) { + StaticBuffer result = KCVSUtil.get(store,KeyValueStoreUtil.getBuffer(i), KeyValueStoreUtil.getBuffer(j), tx); + if (removed.contains(new KeyColumn(i, j))) { + Assert.assertNull(result); + } else { + Assert.assertEquals(values[i][j], KeyValueStoreUtil.getString(result)); + } + } + } + + } + + @Test + public void storeAndRetrieve() throws StorageException { + String[][] values = generateValues(); + log.debug("Loading values..."); + loadValues(values); + //print(values); + log.debug("Checking values..."); + checkValueExistence(values); + checkValues(values); + } + + @Test + public void storeAndRetrievePerformance() throws StorageException { + int multiplier = 4; + int keys = 50*multiplier, columns = 2000; + String[][] values = KeyValueStoreUtil.generateData(keys,columns); + log.debug("Loading values: "+keys+"x"+columns); + long time = System.currentTimeMillis(); + loadValues(values); + System.out.println("Loading time (ms): " + (System.currentTimeMillis()-time)); + //print(values); + Random r = new Random(); + int trials = 500*multiplier; + int delta = 10; + log.debug("Reading values: "+trials+" trials"); + time = System.currentTimeMillis(); + for (int t=0;t deleted = deleteValues(7); + log.debug("Checking values..."); + checkValueExistence(values, deleted); + checkValues(values, deleted); + } + + @Test + public void deleteColumnsTest2() throws StorageException { + String[][] values = generateValues(); + log.debug("Loading values..."); + loadValues(values); + Set deleted = deleteValues(7); + clopen(); + log.debug("Checking values..."); + checkValueExistence(values, deleted); + checkValues(values, deleted); + } + + @Test + public void deleteKeys() throws StorageException { + String[][] values = generateValues(); + log.debug("Loading values..."); + loadValues(values); + Set deleted = deleteKeys(11); + clopen(); + checkKeys(deleted); + } + + @Test + public void scanTest() throws StorageException { + if (manager.getFeatures().supportsScan()) { + String[][] values = generateValues(); + loadValues(values); + RecordIterator iterator0 = store.getKeys(tx); + Assert.assertEquals(numKeys, KeyValueStoreUtil.count(iterator0)); + clopen(); + RecordIterator iterator1 = store.getKeys(tx); + RecordIterator iterator2 = store.getKeys(tx); + // The idea is to open an iterator without using it + // to make sure that closing a transaction will clean it up. + // (important for BerkeleyJE where leaving cursors open causes exceptions) + @SuppressWarnings("unused") + RecordIterator iterator3 = store.getKeys(tx); + Assert.assertEquals(numKeys, KeyValueStoreUtil.count(iterator1)); + Assert.assertEquals(numKeys, KeyValueStoreUtil.count(iterator2)); + } + } + + public void checkSlice(String[][] values, Set removed, int key, int start, int end, int limit) throws StorageException { + List entries; + if (limit <= 0) + entries = store.getSlice(new KeySliceQuery(KeyValueStoreUtil.getBuffer(key), KeyValueStoreUtil.getBuffer(start), KeyValueStoreUtil.getBuffer(end)), tx); + else + entries = store.getSlice(new KeySliceQuery(KeyValueStoreUtil.getBuffer(key), KeyValueStoreUtil.getBuffer(start), KeyValueStoreUtil.getBuffer(end), limit), tx); + + int pos = 0; + for (int i = start; i < end; i++) { + if (removed.contains(new KeyColumn(key, i))) continue; + if (limit <= 0 || pos < limit) { + Assert.assertTrue(entries.size()>pos); + Entry entry = entries.get(pos); + int col = KeyValueStoreUtil.getID(entry.getColumn()); + String str = KeyValueStoreUtil.getString(entry.getValue()); + Assert.assertEquals(i, col); + Assert.assertEquals(values[key][i], str); + } + pos++; + } + Assert.assertNotNull(entries); + if (limit > 0 && pos > limit) Assert.assertEquals(limit, entries.size()); + else Assert.assertEquals(pos, entries.size()); + } + + @Test + public void intervalTest1() throws StorageException { + String[][] values = generateValues(); + log.debug("Loading values..."); + loadValues(values); + Set deleted = Sets.newHashSet(); + clopen(); + int trails = 5000; + for (int t = 0; t < trails; t++) { + int key = RandomGenerator.randomInt(0, numKeys); + int start = RandomGenerator.randomInt(0, numColumns); + int end = RandomGenerator.randomInt(start, numColumns); + int limit = RandomGenerator.randomInt(1, 30); + checkSlice(values, deleted, key, start, end, limit); + checkSlice(values, deleted, key, start, end, -1); + } + } + + @Test + public void intervalTest2() throws StorageException { + String[][] values = generateValues(); + log.debug("Loading values..."); + loadValues(values); + Set deleted = deleteValues(7); + clopen(); + int trails = 5000; + for (int t = 0; t < trails; t++) { + int key = RandomGenerator.randomInt(0, numKeys); + int start = RandomGenerator.randomInt(0, numColumns); + int end = RandomGenerator.randomInt(start, numColumns); + int limit = RandomGenerator.randomInt(1, 30); + checkSlice(values, deleted, key, start, end, limit); + checkSlice(values, deleted, key, start, end, -1); + } + } + + + @Test + public void getNonExistentKeyReturnsNull() throws Exception { + StoreTransaction txn = manager.beginTransaction(ConsistencyLevel.DEFAULT); + Assert.assertEquals(null, KeyColumnValueStoreUtil.get(store, txn, 0, "col0")); + Assert.assertEquals(null, KeyColumnValueStoreUtil.get(store, txn, 0, "col1")); + txn.commit(); + } + + @Test + public void insertingGettingAndDeletingSimpleDataWorks() throws Exception { + StoreTransaction txn = manager.beginTransaction(ConsistencyLevel.DEFAULT); + KeyColumnValueStoreUtil.insert(store, txn, 0, "col0", "val0"); + KeyColumnValueStoreUtil.insert(store, txn, 0, "col1", "val1"); + txn.commit(); + + txn = manager.beginTransaction(ConsistencyLevel.DEFAULT); + Assert.assertEquals("val0", KeyColumnValueStoreUtil.get(store, txn, 0, "col0")); + Assert.assertEquals("val1", KeyColumnValueStoreUtil.get(store, txn, 0, "col1")); + KeyColumnValueStoreUtil.delete(store, txn, 0, "col0"); + KeyColumnValueStoreUtil.delete(store, txn, 0, "col1"); + txn.commit(); + + txn = manager.beginTransaction(ConsistencyLevel.DEFAULT); + Assert.assertEquals(null, KeyColumnValueStoreUtil.get(store, txn, 0, "col0")); + Assert.assertEquals(null, KeyColumnValueStoreUtil.get(store, txn, 0, "col1")); + txn.commit(); + } + +// @Test +// public void getSliceNoLimit() throws Exception { +// CassandraThriftStoreManager manager = new CassandraThriftStoreManager(keyspace); +// CassandraThriftKeyColumnValueStore store = +// manager.openDatabase(dbName); +// +// StoreTransaction txn = manager.beginTransaction(); +// KeyColumnValueStoreUtil.insert(store, txn, "key0", "col0", "val0"); +// KeyColumnValueStoreUtil.insert(store, txn, "key0", "col1", "val1"); +// txn.commit(); +// +// txn = manager.beginTransaction(); +// ByteBuffer key0 = KeyColumnValueStoreUtil.stringToByteBuffer("key0"); +// ByteBuffer col0 = KeyColumnValueStoreUtil.stringToByteBuffer("col0"); +// ByteBuffer col2 = KeyColumnValueStoreUtil.stringToByteBuffer("col2"); +// List entries = store.getSlice(key0, col0, col2, txn); +// assertNotNull(entries); +// assertEquals(2, entries.size()); +// assertEquals("col0", KeyColumnValueStoreUtil.byteBufferToString(entries.get(0).getColumn())); +// assertEquals("val0", KeyColumnValueStoreUtil.byteBufferToString(entries.get(0).getValue())); +// assertEquals("col1", KeyColumnValueStoreUtil.byteBufferToString(entries.get(1).getColumn())); +// assertEquals("val1", KeyColumnValueStoreUtil.byteBufferToString(entries.get(1).getValue())); +// +// txn.commit(); +// +// store.shutdown(); +// manager.shutdown(); +// } + + @Test + public void getSliceRespectsColumnLimit() throws Exception { + StoreTransaction txn = manager.beginTransaction(ConsistencyLevel.DEFAULT); + StaticBuffer key = KeyColumnValueStoreUtil.longToByteBuffer(0); + + final int cols = 1024; + + List entries = new LinkedList(); + for (int i = 0; i < cols; i++) { + StaticBuffer col = KeyColumnValueStoreUtil.longToByteBuffer(i); + entries.add(new StaticBufferEntry(col, col)); + } + store.mutate(key, entries, KeyColumnValueStore.NO_DELETIONS, txn); + txn.commit(); + + txn = manager.beginTransaction(ConsistencyLevel.DEFAULT); + StaticBuffer columnStart = KeyColumnValueStoreUtil.longToByteBuffer(0); + StaticBuffer columnEnd = KeyColumnValueStoreUtil.longToByteBuffer(cols); /* + * When limit is greater than or equal to the matching column count, + * all matching columns must be returned. + */ + List result = + store.getSlice(new KeySliceQuery(key, columnStart, columnEnd, cols), txn); + Assert.assertEquals(cols, result.size()); + Assert.assertEquals(entries, result); + result = + store.getSlice(new KeySliceQuery(key, columnStart, columnEnd, cols + 10), txn); + Assert.assertEquals(cols, result.size()); + Assert.assertEquals(entries, result); + + /* + * When limit is less the matching column count, the columns up to the + * limit (ordered bytewise) must be returned. + */ + result = + store.getSlice(new KeySliceQuery(key, columnStart, columnEnd, cols - 1), txn); + Assert.assertEquals(cols - 1, result.size()); + entries.remove(entries.size() - 1); + Assert.assertEquals(entries, result); + result = + store.getSlice(new KeySliceQuery(key, columnStart, columnEnd, 1), txn); + Assert.assertEquals(1, result.size()); + List firstEntrySingleton = Arrays.asList(entries.get(0)); + Assert.assertEquals(firstEntrySingleton, result); + txn.commit(); + } + + @Test + public void getSliceRespectsAllBoundsInclusionArguments() throws Exception { + // Test case where endColumn=startColumn+1 + StaticBuffer key = KeyColumnValueStoreUtil.longToByteBuffer(0); + StaticBuffer columnBeforeStart = KeyColumnValueStoreUtil.longToByteBuffer(776); + StaticBuffer columnStart = KeyColumnValueStoreUtil.longToByteBuffer(777); + StaticBuffer columnEnd = KeyColumnValueStoreUtil.longToByteBuffer(778); + StaticBuffer columnAfterEnd = KeyColumnValueStoreUtil.longToByteBuffer(779); + + // First insert four test Entries + StoreTransaction txn = manager.beginTransaction(ConsistencyLevel.DEFAULT); + List entries = Arrays.asList( + (Entry)new StaticBufferEntry(columnBeforeStart, columnBeforeStart), + new StaticBufferEntry(columnStart, columnStart), + new StaticBufferEntry(columnEnd, columnEnd), + new StaticBufferEntry(columnAfterEnd, columnAfterEnd)); + store.mutate(key, entries, KeyColumnValueStore.NO_DELETIONS, txn); + txn.commit(); + + // getSlice() with only start inclusive + txn = manager.beginTransaction(ConsistencyLevel.DEFAULT); + List result = store.getSlice(new KeySliceQuery(key, columnStart, columnEnd), txn); + Assert.assertEquals(1, result.size()); + Assert.assertEquals(777, KeyColumnValueStoreUtil.bufferToLong(result.get(0).getColumn())); + txn.commit(); + + } + + + @Test + public void containsKeyReturnsTrueOnExtantKey() throws Exception { + StaticBuffer key1 = KeyColumnValueStoreUtil.longToByteBuffer(1); + StoreTransaction txn = manager.beginTransaction(ConsistencyLevel.DEFAULT); + Assert.assertFalse(store.containsKey(key1, txn)); + KeyColumnValueStoreUtil.insert(store, txn, 1, "c", "v"); + txn.commit(); + + txn = manager.beginTransaction(ConsistencyLevel.DEFAULT); + Assert.assertTrue(store.containsKey(key1, txn)); + txn.commit(); + } + + + @Test + public void containsKeyReturnsFalseOnNonexistentKey() throws Exception { + StoreTransaction txn = manager.beginTransaction(ConsistencyLevel.DEFAULT); + StaticBuffer key1 = KeyColumnValueStoreUtil.longToByteBuffer(1); + Assert.assertFalse(store.containsKey(key1, txn)); + txn.commit(); + } + + + @Test + public void containsKeyColumnReturnsFalseOnNonexistentInput() throws Exception { + StoreTransaction txn = manager.beginTransaction(ConsistencyLevel.DEFAULT); + StaticBuffer key1 = KeyColumnValueStoreUtil.longToByteBuffer(1); + StaticBuffer c = KeyColumnValueStoreUtil.stringToByteBuffer("c"); + Assert.assertFalse(KCVSUtil.containsKeyColumn(store,key1, c, txn)); + txn.commit(); + } + + @Test + public void containsKeyColumnReturnsTrueOnExtantInput() throws Exception { + StoreTransaction txn = manager.beginTransaction(ConsistencyLevel.DEFAULT); + KeyColumnValueStoreUtil.insert(store, txn, 1, "c", "v"); + txn.commit(); + + txn = manager.beginTransaction(ConsistencyLevel.DEFAULT); + StaticBuffer key1 = KeyColumnValueStoreUtil.longToByteBuffer(1); + StaticBuffer c = KeyColumnValueStoreUtil.stringToByteBuffer("c"); + Assert.assertTrue(KCVSUtil.containsKeyColumn(store,key1, c, txn)); + txn.commit(); + } +} + diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/MultiWriteKeyColumnValueStoreTest.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/MultiWriteKeyColumnValueStoreTest.java new file mode 100644 index 0000000000..0b960f8d7e --- /dev/null +++ b/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/MultiWriteKeyColumnValueStoreTest.java @@ -0,0 +1,414 @@ +package com.thinkaurelius.titan.diskstorage.accumulo; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.thinkaurelius.titan.diskstorage.KeyColumnValueStoreUtil; +import com.thinkaurelius.titan.diskstorage.StaticBuffer; +import com.thinkaurelius.titan.diskstorage.StorageException; +import com.thinkaurelius.titan.diskstorage.keycolumnvalue.*; +import com.thinkaurelius.titan.diskstorage.util.StaticArrayBuffer; + +import static com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyColumnValueStore.*; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; + +public abstract class MultiWriteKeyColumnValueStoreTest { + + private Logger log = LoggerFactory.getLogger(MultiWriteKeyColumnValueStoreTest.class); + + int numKeys = 50; + int numColumns = 5; + + int bufferSize = 20; + + protected String storeName1 = "testStore1"; + private KeyColumnValueStore store1; + protected String storeName2 = "testStore2"; + private KeyColumnValueStore store2; + + + public KeyColumnValueStoreManager manager; + public StoreTransaction tx; + + + private Random rand = new Random(10); + + @Before + public void setUp() throws Exception { + openStorageManager().clearStorage(); + open(); + } + + public abstract KeyColumnValueStoreManager openStorageManager() throws StorageException; + + public void open() throws StorageException { + manager = openStorageManager(); + Assert.assertTrue(manager.getFeatures().supportsBatchMutation()); + tx = new BufferTransaction(manager.beginTransaction(ConsistencyLevel.DEFAULT), manager, bufferSize, 1, 0); + store1 = new BufferedKeyColumnValueStore(manager.openDatabase(storeName1), true); + store2 = new BufferedKeyColumnValueStore(manager.openDatabase(storeName2), true); + + } + + public void clopen() throws StorageException { + close(); + open(); + } + + @After + public void tearDown() throws Exception { + close(); + } + + public void close() throws StorageException { + if (tx != null) tx.commit(); + if (null != store1) store1.close(); + if (null != store2) store2.close(); + if (null != manager) manager.close(); + } + + @Test + public void deletionsAppliedBeforeAdditions() throws StorageException { + + StaticBuffer b1 = KeyColumnValueStoreUtil.longToByteBuffer(1); + + Assert.assertNull(KCVSUtil.get(store1,b1, b1, tx)); + + List additions = Arrays.asList(new StaticBufferEntry(b1, b1)); + + List deletions = Arrays.asList(b1); + + Map combination = new HashMap(1); + Map deleteOnly = new HashMap(1); + Map addOnly = new HashMap(1); + + combination.put(b1, new KCVMutation(additions, deletions)); + deleteOnly.put(b1, new KCVMutation(KeyColumnValueStore.NO_ADDITIONS, deletions)); + addOnly.put(b1, new KCVMutation(additions, KeyColumnValueStore.NO_DELETIONS)); + + store1.mutate(b1, additions, deletions, tx); + tx.flush(); + + StaticBuffer result = KCVSUtil.get(store1,b1, b1, tx); + + Assert.assertEquals(b1, result); + + store1.mutate(b1, NO_ADDITIONS, deletions, tx); + tx.flush(); + + for (int i = 0; i < 100; i++) { + StaticBuffer n = KCVSUtil.get(store1,b1, b1, tx); + Assert.assertNull(n); + store1.mutate(b1, additions, NO_DELETIONS, tx); + tx.flush(); + store1.mutate(b1, NO_ADDITIONS, deletions, tx); + tx.flush(); + n = KCVSUtil.get(store1,b1, b1, tx); + Assert.assertNull(n); + } + + for (int i = 0; i < 100; i++) { + store1.mutate(b1, NO_ADDITIONS, deletions, tx); + tx.flush(); + store1.mutate(b1, additions, NO_DELETIONS, tx); + tx.flush(); + Assert.assertEquals(b1, KCVSUtil.get(store1,b1, b1, tx)); + } + + for (int i = 0; i < 100; i++) { + store1.mutate(b1, additions, deletions, tx); + tx.flush(); + Assert.assertEquals(b1, KCVSUtil.get(store1,b1, b1, tx)); + } + } + + @Test + public void mutateManyWritesSameKeyOnMultipleCFs() throws StorageException { + + final long arbitraryLong = 42; + assert 0 < arbitraryLong; + + final StaticBuffer key = KeyColumnValueStoreUtil.longToByteBuffer(arbitraryLong * arbitraryLong); + final StaticBuffer val = KeyColumnValueStoreUtil.longToByteBuffer(arbitraryLong * arbitraryLong * arbitraryLong); + final StaticBuffer col = KeyColumnValueStoreUtil.longToByteBuffer(arbitraryLong); + final StaticBuffer nextCol = KeyColumnValueStoreUtil.longToByteBuffer(arbitraryLong + 1); + + final StoreTransaction directTx = manager.beginTransaction(ConsistencyLevel.DEFAULT); + + KCVMutation km = new KCVMutation( + ImmutableList.of(new StaticBufferEntry(col, val)), + ImmutableList.of()); + + Map keyColumnAndValue = ImmutableMap.of(key, km); + + Map> mutations = + ImmutableMap.of( + storeName1, keyColumnAndValue, + storeName2, keyColumnAndValue); + + manager.mutateMany(mutations, directTx); + + directTx.commit(); + + KeySliceQuery query = new KeySliceQuery(key, col, nextCol); + List expected = + ImmutableList.of(new StaticBufferEntry(col, val)); + + Assert.assertEquals(expected, store1.getSlice(query, tx)); + Assert.assertEquals(expected, store2.getSlice(query, tx)); + + } + + @Test + public void mutateManyStressTest() throws StorageException { + + Map> state = + new HashMap>(); + + int dels = 1024; + int adds = 4096; + + for (int round = 0; round < 5; round++) { + Map changes = mutateState(state, dels, adds); + + applyChanges(changes, store1, tx); + applyChanges(changes, store2, tx); + tx.flush(); + + int deletesExpected = 0 == round ? 0 : dels; + + int stateSizeExpected = adds + (adds - dels) * round; + + Assert.assertEquals(stateSizeExpected, checkThatStateExistsInStore(state, store1, round)); + Assert.assertEquals(deletesExpected, checkThatDeletionsApplied(changes, store1, round)); + + Assert.assertEquals(stateSizeExpected, checkThatStateExistsInStore(state, store2, round)); + Assert.assertEquals(deletesExpected, checkThatDeletionsApplied(changes, store2, round)); + } + } + + public void applyChanges(Map changes, KeyColumnValueStore store, StoreTransaction tx) throws StorageException { + for (Map.Entry change : changes.entrySet()) { + store.mutate(change.getKey(), change.getValue().getAdditions(), change.getValue().getDeletions(), tx); + } + } + + public int checkThatStateExistsInStore(Map> state, KeyColumnValueStore store, int round) throws StorageException { + int checked = 0; + + for (StaticBuffer key : state.keySet()) { + for (StaticBuffer col : state.get(key).keySet()) { + StaticBuffer val = state.get(key).get(col); + + Assert.assertEquals(val, KCVSUtil.get(store,key, col, tx)); + + checked++; + } + } + + log.debug("Checked existence of {} key-column-value triples on round {}", checked, round); + + return checked; + } + + public int checkThatDeletionsApplied(Map changes, KeyColumnValueStore store, int round) throws StorageException { + int checked = 0; + int skipped = 0; + + for (StaticBuffer key : changes.keySet()) { + KCVMutation m = changes.get(key); + + if (!m.hasDeletions()) + continue; + + List deletions = m.getDeletions(); + + List additions = m.getAdditions(); + + for (StaticBuffer col : deletions) { + + if (null != additions && additions.contains(new StaticBufferEntry(col, col))) { + skipped++; + continue; + } + + Assert.assertNull(KCVSUtil.get(store,key, col, tx)); + + checked++; + } + } + + log.debug("Checked absence of {} key-column-value deletions on round {} (skipped {})", new Object[]{checked, round, skipped}); + + return checked; + } + + /** + * Pseudorandomly change the supplied {@code state}. + *

+ * This method removes {@code min(maxDeletionCount, S)} entries from the + * maps in {@code state.values()}, where {@code S} is the sum of the sizes + * of the maps in {@code state.values()}; this method then adds + * {@code additionCount} pseudorandomly generated entries spread across + * {@code state.values()}, potentially adding new keys to {@code state} + * since they are randomly generated. This method then returns a map of keys + * to Mutations representing the changes it has made to {@code state}. + * + * @param state Maps keys -> columns -> values + * @param maxDeletionCount Remove at most this many entries from state + * @param additionCount Add exactly this many entries to state + * @return A KCVMutation map + */ + public Map mutateState( + Map> state, + int maxDeletionCount, int additionCount) { + + final int keyLength = 8; + final int colLength = 16; + + Map result = new HashMap(); + + // deletion pass + int dels = 0; + + StaticBuffer key = null, col = null; + + Iterator keyIter = state.keySet().iterator(); + + while (keyIter.hasNext() && dels < maxDeletionCount) { + key = keyIter.next(); + + Iterator colIter = + state.get(key).keySet().iterator(); + + while (colIter.hasNext() && dels < maxDeletionCount) { + col = colIter.next(); + + if (!result.containsKey(key)) { + KCVMutation m = new KCVMutation(new LinkedList(), + new LinkedList()); + result.put(key, m); + } + + result.get(key).deletion(col); + + dels++; + + colIter.remove(); + + if (state.get(key).isEmpty()) { + assert !colIter.hasNext(); + keyIter.remove(); + } + } + } + + // addition pass + for (int i = 0; i < additionCount; i++) { + + while (true) { + byte keyBuf[] = new byte[keyLength]; + rand.nextBytes(keyBuf); + key = new StaticArrayBuffer(keyBuf); + + byte colBuf[] = new byte[colLength]; + rand.nextBytes(colBuf); + col = new StaticArrayBuffer(colBuf); + + if (!state.containsKey(key) || !state.get(key).containsKey(col)) { + break; + } + } + + if (!state.containsKey(key)) { + Map m = new HashMap(); + state.put(key, m); + } + + state.get(key).put(col, col); + + if (!result.containsKey(key)) { + KCVMutation m = new KCVMutation(new LinkedList(), + new LinkedList()); + result.put(key, m); + } + + result.get(key).addition(new StaticBufferEntry(col, col)); + + } + + return result; + } + + public Map generateMutation(int keyCount, int columnCount, Map deleteFrom) { + Map result = new HashMap(keyCount); + + Random keyRand = new Random(keyCount); + Random colRand = new Random(columnCount); + + final int keyLength = 8; + final int colLength = 6; + + Iterator> deleteIter = null; + List lastDeleteIterResult = null; + + if (null != deleteFrom) { + deleteIter = deleteFrom.entrySet().iterator(); + } + + for (int ik = 0; ik < keyCount; ik++) { + byte keyBuf[] = new byte[keyLength]; + keyRand.nextBytes(keyBuf); + StaticBuffer key = new StaticArrayBuffer(keyBuf); + + List additions = new LinkedList(); + List deletions = new LinkedList(); + + for (int ic = 0; ic < columnCount; ic++) { + + boolean deleteSucceeded = false; + if (null != deleteIter && 1 == ic % 2) { + + if (null == lastDeleteIterResult || lastDeleteIterResult.isEmpty()) { + while (deleteIter.hasNext()) { + Map.Entry ent = deleteIter.next(); + if (ent.getValue().hasAdditions() && !ent.getValue().getAdditions().isEmpty()) { + lastDeleteIterResult = ent.getValue().getAdditions(); + break; + } + } + } + + + if (null != lastDeleteIterResult && !lastDeleteIterResult.isEmpty()) { + Entry e = lastDeleteIterResult.get(0); + lastDeleteIterResult.remove(0); + deletions.add(e.getColumn()); + deleteSucceeded = true; + } + } + + if (!deleteSucceeded) { + byte colBuf[] = new byte[colLength]; + colRand.nextBytes(colBuf); + StaticBuffer col = new StaticArrayBuffer(colBuf); + + additions.add(new StaticBufferEntry(col, col)); + } + + } + + KCVMutation m = new KCVMutation(additions, deletions); + + result.put(key, m); + } + + return result; + } +} From 03766e08ee56fa83aef2beb2b0b5eff366f1912b Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Mon, 10 Jun 2013 11:36:00 -0400 Subject: [PATCH 04/50] Hard-wired local accumulo instance, passes all unit tests except AccumuloGraphConcurrentTest. --- pom.xml | 32 +++++++++- titan-accumulo/pom.xml | 8 +++ .../accumulo/AccumuloKeyColumnValueStore.java | 40 +++--------- .../accumulo/AccumuloStoreManager.java | 64 +++++++++++-------- .../titan/diskstorage/accumulo/Bytes.java | 31 +++++++++ .../diskstorage/accumulo/TestAccumulo.java | 44 ------------- .../titan/AccumuloStorageSetup.java | 28 ++++++-- .../blueprints/AccumuloBlueprintsTest.java | 6 +- titan-berkeleyje/pom.xml | 8 +++ titan-cassandra/pom.xml | 26 +++++--- titan-es/pom.xml | 8 +++ titan-hbase/pom.xml | 8 +++ titan-lucene/pom.xml | 8 +++ titan-test/pom.xml | 8 +++ 14 files changed, 198 insertions(+), 121 deletions(-) create mode 100644 titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/Bytes.java delete mode 100644 titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/TestAccumulo.java diff --git a/pom.xml b/pom.xml index dc3f603428..6592cea577 100644 --- a/pom.xml +++ b/pom.xml @@ -101,9 +101,9 @@ maven-javadoc-plugin 2.8 - - com.thinkaurelius.titan.diskstorage;com.thinkaurelius.titan.graphdb;com.thinkaurelius.titan.tinkerpop;com.thinkaurelius.titan.util - + + com.thinkaurelius.titan.diskstorage;com.thinkaurelius.titan.graphdb;com.thinkaurelius.titan.tinkerpop;com.thinkaurelius.titan.util + @@ -147,6 +147,32 @@ random + + org.apache.maven.plugins + maven-surefire-report-plugin + 2.14.1 + + + ${project.build.directory}/surefire-reports + + + true + + + + + test + + report + + + + + + org.apache.maven.plugins + maven-jxr-plugin + 2.3 + diff --git a/titan-accumulo/pom.xml b/titan-accumulo/pom.xml index 13c87ca0b0..333e2c67bf 100644 --- a/titan-accumulo/pom.xml +++ b/titan-accumulo/pom.xml @@ -114,6 +114,14 @@ + + org.apache.maven.plugins + maven-surefire-report-plugin + + + org.apache.maven.plugins + maven-jxr-plugin + maven-assembly-plugin 2.4 diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java index 30382a45c4..f7fdfc142e 100644 --- a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java +++ b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java @@ -79,25 +79,24 @@ public void close() throws StorageException { @Override public boolean containsKey(StaticBuffer key, StoreTransaction txh) throws StorageException { - byte[] keyBytes = key.as(StaticBuffer.ARRAY_FACTORY); + Scanner scanner; try { - Scanner scanner = connector.createScanner(tableName, DEFAULT_AUTHORIZATIONS); - - scanner.setRange(new Range(new Text(keyBytes))); - scanner.fetchColumnFamily(new Text(columnFamilyBytes)); - - return scanner.iterator().hasNext(); - + scanner = connector.createScanner(tableName, DEFAULT_AUTHORIZATIONS); } catch (TableNotFoundException ex) { logger.error("Should never throw this exception!", ex); throw new PermanentStorageException(ex); } + + byte[] keyBytes = key.as(StaticBuffer.ARRAY_FACTORY); + scanner.setRange(new Range(new Text(keyBytes))); + scanner.fetchColumnFamily(new Text(columnFamilyBytes)); + + return scanner.iterator().hasNext(); } @Override public List getSlice(KeySliceQuery query, StoreTransaction txh) throws StorageException { Scanner scanner; - try { scanner = connector.createScanner(tableName, DEFAULT_AUTHORIZATIONS); } catch (TableNotFoundException ex) { @@ -121,8 +120,8 @@ public List getSlice(KeySliceQuery query, StoreTransaction txh) throws St byte[] cqBytes = entry.getKey().getColumnQualifier().getBytes(); if (count < query.getLimit()) { - if (startBytes.length == 0 || byteCompare(startBytes, cqBytes) <= 0) { - if (endBytes.length == 0 || byteCompare(cqBytes, endBytes) < 0) { + if (startBytes.length == 0 || Bytes.compare(startBytes, cqBytes) <= 0) { + if (endBytes.length == 0 || Bytes.compare(cqBytes, endBytes) < 0) { entries.add(StaticBufferEntry.of(new StaticArrayBuffer(cqBytes), new StaticArrayBuffer(entry.getValue().get()))); count++; @@ -134,25 +133,6 @@ public List getSlice(KeySliceQuery query, StoreTransaction txh) throws St return entries; } - private static int byteCompare(final byte[] left, final byte[] right) { - return byteCompare(left, 0, left.length, right, 0, right.length); - } - - private static int byteCompare(byte[] buffer1, int offset1, int length1, - byte[] buffer2, int offset2, int length2) { - // Bring WritableComparator code local - int end1 = offset1 + length1; - int end2 = offset2 + length2; - for (int i = offset1, j = offset2; i < end1 && j < end2; i++, j++) { - int a = (buffer1[i] & 0xff); - int b = (buffer2[j] & 0xff); - if (a != b) { - return a - b; - } - } - return length1 - length2; - } - @Override public void mutate(StaticBuffer key, List additions, List deletions, StoreTransaction txh) throws StorageException { diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java index aa584da089..3f1b7ff536 100644 --- a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java +++ b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java @@ -29,6 +29,7 @@ import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.ZooKeeperInstance; import org.apache.accumulo.core.client.admin.TableOperations; +import org.apache.accumulo.core.client.mock.MockAccumulo; import org.apache.accumulo.core.client.mock.MockInstance; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Range; @@ -45,39 +46,49 @@ public class AccumuloStoreManager extends DistributedStoreManager implements KeyColumnValueStoreManager { private static final Logger logger = LoggerFactory.getLogger(AccumuloStoreManager.class); + // Configuration namespace + public static final String ACCUMULO_CONFIGURATION_NAMESPACE = "accumulo-config"; + // Default deleter, scanner, writer parameters private static final Authorizations DEFAULT_AUTHORIZATIONS = new Authorizations(); private static final Long DEFAULT_MAX_MEMORY = 50 * 1024 * 1024l; private static final Long DEFAULT_MAX_LATENCY = 2 * 60 * 1000l; private static final Long DEFAULT_TIMEOUT = Long.MAX_VALUE; private static final Integer DEFAULT_MAX_QUERY_THREADS = 3; private static final Integer DEFAULT_MAX_WRITE_THREADS = 3; + // Configuration keys + public static final String ACCUMULO_INTSANCE_KEY = "instance"; + public static final String ACCUMULO_ZOOKEEPERS_KEY = "zookeepers"; + public static final String ACCUMULO_USER_KEY = "username"; + public static final String ACCUMULO_PASSWORD_KEY = "password"; public static final String TABLE_NAME_KEY = "tablename"; + // Configuration defaults public static final String TABLE_NAME_DEFAULT = "titan"; public static final int PORT_DEFAULT = 9160; - public static final String ACCUMULO_CONFIGURATION_NAMESPACE = "accumulo-config"; - public static final ImmutableMap ACCUMULO_CONFIGURATION; - - static { - ACCUMULO_CONFIGURATION = new ImmutableMap.Builder() - .put(GraphDatabaseConfiguration.HOSTNAME_KEY, "accumulo.zookeeper.quorum") - .put(GraphDatabaseConfiguration.PORT_KEY, "accumulo.zookeeper.property.clientPort") - .build(); - } + // Instance variables private final String tableName; - private final ConcurrentMap openStores; + private final String instanceName; + private final String zooKeepers; + private final String username; + private final String password; private final Instance instance; private final Connector connector; + private final ConcurrentMap openStores; private final StoreFeatures features; public AccumuloStoreManager(Configuration config) throws StorageException { super(config, PORT_DEFAULT); - this.tableName = config.getString(TABLE_NAME_KEY, TABLE_NAME_DEFAULT); + tableName = config.getString(TABLE_NAME_KEY, TABLE_NAME_DEFAULT); - openStores = new ConcurrentHashMap(); + // Accumulo specific keys + Configuration accumuloConfig = config.subset(ACCUMULO_CONFIGURATION_NAMESPACE); + instanceName = accumuloConfig.getString(ACCUMULO_INTSANCE_KEY); + zooKeepers = accumuloConfig.getString(ACCUMULO_ZOOKEEPERS_KEY); + + username = accumuloConfig.getString(ACCUMULO_USER_KEY); + password = accumuloConfig.getString(ACCUMULO_PASSWORD_KEY); instance = new ZooKeeperInstance("EtCloud", "localhost"); - // instance = new MockInstance("EtCloud"); try { connector = instance.getConnector("root", "bobross".getBytes()); } catch (AccumuloException ex) { @@ -88,7 +99,8 @@ public AccumuloStoreManager(Configuration config) throws StorageException { throw new PermanentStorageException(ex.getMessage(), ex); } - // TODO: allowing publicly mutate fields is bad, should be fixed + openStores = new ConcurrentHashMap(); + features = new StoreFeatures(); features.supportsScan = true; features.supportsBatchMutation = true; @@ -319,17 +331,19 @@ public String getConfigurationProperty(final String key) throws StorageException @Override public void setConfigurationProperty(final String key, final String value) throws StorageException { ensureTableExists(tableName); - - TableOperations operations = connector.tableOperations(); - try { - operations.setProperty(tableName, key, value); - } catch (AccumuloException ex) { - logger.error(ex.getMessage(), ex); - throw new PermanentStorageException(ex); - } catch (AccumuloSecurityException ex) { - logger.error(ex.getMessage(), ex); - throw new PermanentStorageException(ex); - } + /* + + TableOperations operations = connector.tableOperations(); + try { + operations.setProperty(tableName, key, value); + } catch (AccumuloException ex) { + logger.error(ex.getMessage(), ex); + throw new PermanentStorageException(ex); + } catch (AccumuloSecurityException ex) { + logger.error(ex.getMessage(), ex); + throw new PermanentStorageException(ex); + } + */ } private static void waitUntil(long until) { diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/Bytes.java b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/Bytes.java new file mode 100644 index 0000000000..3433057efb --- /dev/null +++ b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/Bytes.java @@ -0,0 +1,31 @@ +/* + * To change this template, choose Tools | Templates + * and open the template in the editor. + */ +package com.thinkaurelius.titan.diskstorage.accumulo; + +/** + * + * @author edeprit + */ +public class Bytes { + + public static int compare(final byte[] left, final byte[] right) { + return compare(left, 0, left.length, right, 0, right.length); + } + + public static int compare(byte[] buffer1, int offset1, int length1, + byte[] buffer2, int offset2, int length2) { + // Bring WritableComparator code local + int end1 = offset1 + length1; + int end2 = offset2 + length2; + for (int i = offset1, j = offset2; i < end1 && j < end2; i++, j++) { + int a = (buffer1[i] & 0xff); + int b = (buffer2[j] & 0xff); + if (a != b) { + return a - b; + } + } + return length1 - length2; + } +} diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/TestAccumulo.java b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/TestAccumulo.java deleted file mode 100644 index 6113606f44..0000000000 --- a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/TestAccumulo.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * To change this template, choose Tools | Templates - * and open the template in the editor. - */ -package com.thinkaurelius.titan.diskstorage.accumulo; - -import com.thinkaurelius.titan.diskstorage.StaticBuffer; -import com.thinkaurelius.titan.diskstorage.keycolumnvalue.ConsistencyLevel; -import com.thinkaurelius.titan.diskstorage.keycolumnvalue.Entry; -import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyColumnValueStore; -import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyColumnValueStoreManager; -import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeySliceQuery; -import com.thinkaurelius.titan.diskstorage.keycolumnvalue.StaticBufferEntry; -import com.thinkaurelius.titan.diskstorage.keycolumnvalue.StoreTransaction; -import com.thinkaurelius.titan.diskstorage.util.ByteBufferUtil; -import com.thinkaurelius.titan.diskstorage.util.StaticArrayBuffer; -import java.util.Collections; -import org.apache.commons.configuration.BaseConfiguration; - -/** - * - * @author edeprit - */ -public class TestAccumulo { - - public static void main(String[] args) throws Exception { - KeyColumnValueStoreManager manager = new AccumuloStoreManager(new BaseConfiguration()); - KeyColumnValueStore store = manager.openDatabase("foo"); - - StoreTransaction txh = manager.beginTransaction(ConsistencyLevel.DEFAULT); - - Entry entry = new StaticBufferEntry(ByteBufferUtil.getIntBuffer(2), ByteBufferUtil.getIntBuffer(3)); - - StaticBuffer key = ByteBufferUtil.getIntBuffer(1); - - store.mutate(key, Collections.singletonList(entry), - KeyColumnValueStore.NO_DELETIONS, txh); - - KeySliceQuery query = new KeySliceQuery(key, new StaticArrayBuffer(new byte[0]), new StaticArrayBuffer(new byte[0])); - for (Entry e : store.getSlice(query, txh)) { - System.out.println(e); - } - } -} diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java index 2ffb1dba3b..cd76d9eb49 100644 --- a/titan-accumulo/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java +++ b/titan-accumulo/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java @@ -1,5 +1,6 @@ package com.thinkaurelius.titan; +import com.thinkaurelius.titan.diskstorage.accumulo.AccumuloStoreManager; import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration; import org.apache.commons.configuration.BaseConfiguration; import org.apache.commons.configuration.Configuration; @@ -9,13 +10,12 @@ import java.io.IOException; public class AccumuloStorageSetup { + private static Process ACCUMULO = null; // amount of seconds to wait before assuming that HBase shutdown private static final int SHUTDOWN_TIMEOUT_SEC = 20; - // hbase config for testing private static final String ACCUMULO_CONFIG_DIR = "./src/test/config"; - // default pid file location private static final String ACCUMULO_PID_FILE = "/tmp/accumulo-" + System.getProperty("user.name") + "-master.pid"; @@ -27,11 +27,13 @@ public class AccumuloStorageSetup { File accumuloRoot = new File("./src/test/titan-accumulo-test-data"); File zookeeperDataDir = new File("./src/test/titan-zookeeper-test"); - if (accumuloRoot.exists()) + if (accumuloRoot.exists()) { FileUtils.deleteDirectory(accumuloRoot); + } - if (zookeeperDataDir.exists()) + if (zookeeperDataDir.exists()) { FileUtils.deleteDirectory(zookeeperDataDir); + } } catch (IOException e) { System.err.println("Failed to delete old Accumulo test directories: '" + e.getMessage() + "', ignoring."); } @@ -51,14 +53,26 @@ public void run() { public static Configuration getAccumuloStorageConfiguration() { BaseConfiguration config = new BaseConfiguration(); - config.addProperty(GraphDatabaseConfiguration.STORAGE_BACKEND_KEY, "com.thinkaurelius.titan.diskstorage.accumulo.AccumuloStoreManager"); + config.addProperty(GraphDatabaseConfiguration.STORAGE_BACKEND_KEY, + "com.thinkaurelius.titan.diskstorage.accumulo.AccumuloStoreManager"); return config; } public static Configuration getAccumuloGraphConfiguration() { - BaseConfiguration config = new BaseConfiguration(); - config.subset(GraphDatabaseConfiguration.STORAGE_NAMESPACE).addProperty(GraphDatabaseConfiguration.STORAGE_BACKEND_KEY, + Configuration config = new BaseConfiguration(); + Configuration storageConfig = config.subset(GraphDatabaseConfiguration.STORAGE_NAMESPACE); + + storageConfig.addProperty(GraphDatabaseConfiguration.STORAGE_BACKEND_KEY, "com.thinkaurelius.titan.diskstorage.accumulo.AccumuloStoreManager"); + + Configuration accumuloConfig = storageConfig.subset(AccumuloStoreManager.ACCUMULO_CONFIGURATION_NAMESPACE); + + accumuloConfig.addProperty(AccumuloStoreManager.ACCUMULO_INTSANCE_KEY, "EtCloud"); + accumuloConfig.addProperty(AccumuloStoreManager.ACCUMULO_ZOOKEEPERS_KEY, "localhost"); + + accumuloConfig.addProperty(AccumuloStoreManager.ACCUMULO_USER_KEY, "root"); + accumuloConfig.addProperty(AccumuloStoreManager.ACCUMULO_PASSWORD_KEY, "bobross"); + return config; } diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/blueprints/AccumuloBlueprintsTest.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/blueprints/AccumuloBlueprintsTest.java index 0c04522f75..92040147f0 100644 --- a/titan-accumulo/src/test/java/com/thinkaurelius/titan/blueprints/AccumuloBlueprintsTest.java +++ b/titan-accumulo/src/test/java/com/thinkaurelius/titan/blueprints/AccumuloBlueprintsTest.java @@ -8,10 +8,10 @@ import com.tinkerpop.blueprints.Graph; import java.io.IOException; + /** * @author Matthias Broecheler (me@matthiasb.com) */ - public class AccumuloBlueprintsTest extends TitanBlueprintsTest { @Override @@ -36,8 +36,8 @@ public Graph generateGraph() { @Override public void cleanUp() throws StorageException { AccumuloStoreManager s = new AccumuloStoreManager(AccumuloStorageSetup - .getAccumuloGraphConfiguration() - .subset(GraphDatabaseConfiguration.STORAGE_NAMESPACE)); + .getAccumuloGraphConfiguration() + .subset(GraphDatabaseConfiguration.STORAGE_NAMESPACE)); s.clearStorage(); } diff --git a/titan-berkeleyje/pom.xml b/titan-berkeleyje/pom.xml index 14aa7e7fda..041dff71fe 100644 --- a/titan-berkeleyje/pom.xml +++ b/titan-berkeleyje/pom.xml @@ -104,6 +104,14 @@ + + org.apache.maven.plugins + maven-surefire-report-plugin + + + org.apache.maven.plugins + maven-jxr-plugin + maven-assembly-plugin 2.4 diff --git a/titan-cassandra/pom.xml b/titan-cassandra/pom.xml index bb8148e6b2..374c8061fb 100644 --- a/titan-cassandra/pom.xml +++ b/titan-cassandra/pom.xml @@ -43,8 +43,8 @@ net.java.dev.jna jna 3.2.7 - true - + true + - + org.apache.cassandra cassandra-all 1.2.2 @@ -107,12 +107,12 @@ **/*Suite.java + Internal and External Cassandra tests + to run in the same suite. Otherwise, + when an Internal runs before an External, + the Internal Cassandra daemon will hold + all its ports open and the External + Cassandra daemon will fail to bind them. --> always @@ -149,6 +149,14 @@ always + + org.apache.maven.plugins + maven-surefire-report-plugin + + + org.apache.maven.plugins + maven-jxr-plugin + maven-resources-plugin 2.5 diff --git a/titan-es/pom.xml b/titan-es/pom.xml index 022350bbce..a79cce5533 100644 --- a/titan-es/pom.xml +++ b/titan-es/pom.xml @@ -91,6 +91,14 @@ + + org.apache.maven.plugins + maven-surefire-report-plugin + + + org.apache.maven.plugins + maven-jxr-plugin + \ No newline at end of file diff --git a/titan-hbase/pom.xml b/titan-hbase/pom.xml index 3b2586e53f..77a8b572cf 100644 --- a/titan-hbase/pom.xml +++ b/titan-hbase/pom.xml @@ -120,6 +120,14 @@ + + org.apache.maven.plugins + maven-surefire-report-plugin + + + org.apache.maven.plugins + maven-jxr-plugin + maven-assembly-plugin 2.4 diff --git a/titan-lucene/pom.xml b/titan-lucene/pom.xml index 1ffae1dee7..e544f2f373 100644 --- a/titan-lucene/pom.xml +++ b/titan-lucene/pom.xml @@ -101,6 +101,14 @@ + + org.apache.maven.plugins + maven-surefire-report-plugin + + + org.apache.maven.plugins + maven-jxr-plugin + \ No newline at end of file diff --git a/titan-test/pom.xml b/titan-test/pom.xml index 6a43a6356d..afefef9685 100644 --- a/titan-test/pom.xml +++ b/titan-test/pom.xml @@ -62,6 +62,14 @@ -Xms256m -Xmx756m + + org.apache.maven.plugins + maven-surefire-report-plugin + + + org.apache.maven.plugins + maven-jxr-plugin + \ No newline at end of file From 5a3609fe244d019c5ec3a66a5aad47beff3f0895 Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Wed, 12 Jun 2013 13:59:04 -0400 Subject: [PATCH 05/50] Passes all unit tests except AccumuloGraphConcurrentTest with hard-wired local instance. --- pom.xml | 8 ----- titan-accumulo/pom.xml | 1 + .../accumulo/AccumuloKeyColumnValueStore.java | 7 ++--- .../accumulo/AccumuloStoreManager.java | 31 ++++++++++++------- .../titan/AccumuloStorageSetup.java | 20 ++++++++---- .../accumulo/AccumuloKeyColumnValueTest.java | 13 +++----- ...muloMultiWriteKeyColumnValueStoreTest.java | 10 ++---- titan-berkeleyje/pom.xml | 4 --- titan-cassandra/pom.xml | 4 --- titan-es/pom.xml | 4 --- titan-hbase/pom.xml | 4 --- titan-lucene/pom.xml | 4 --- titan-test/pom.xml | 4 --- 13 files changed, 45 insertions(+), 69 deletions(-) diff --git a/pom.xml b/pom.xml index 6592cea577..3f28592498 100644 --- a/pom.xml +++ b/pom.xml @@ -155,9 +155,6 @@ ${project.build.directory}/surefire-reports - - true - @@ -168,11 +165,6 @@ - - org.apache.maven.plugins - maven-jxr-plugin - 2.3 - diff --git a/titan-accumulo/pom.xml b/titan-accumulo/pom.xml index 333e2c67bf..2e1cb52060 100644 --- a/titan-accumulo/pom.xml +++ b/titan-accumulo/pom.xml @@ -58,6 +58,7 @@ **/*Test.java + **/AccumuloGraphConcurrentTest.java **/Internal*.java **/*Suite.java diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java index f7fdfc142e..32e8f5929c 100644 --- a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java +++ b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java @@ -56,10 +56,9 @@ public class AccumuloKeyColumnValueStore implements KeyColumnValueStore { private static final Logger logger = LoggerFactory.getLogger(AccumuloKeyColumnValueStore.class); private static final Authorizations DEFAULT_AUTHORIZATIONS = new Authorizations(); private static final Long DEFAULT_MAX_MEMORY = 50 * 1024 * 1024l; - private static final Long DEFAULT_MAX_LATENCY = 2 * 60 * 1000l; - private static final Long DEFAULT_TIMEOUT = Long.MAX_VALUE; - private static final Integer DEFAULT_MAX_QUERY_THREADS = 3; - private static final Integer DEFAULT_MAX_WRITE_THREADS = 3; + private static final Long DEFAULT_MAX_LATENCY = 100l; + private static final Integer DEFAULT_MAX_QUERY_THREADS = 10; + private static final Integer DEFAULT_MAX_WRITE_THREADS = 10; private final Connector connector; private final String tableName; private final String columnFamily; diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java index 3f1b7ff536..08e62c1660 100644 --- a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java +++ b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java @@ -1,6 +1,5 @@ package com.thinkaurelius.titan.diskstorage.accumulo; -import com.google.common.collect.ImmutableMap; import com.thinkaurelius.titan.diskstorage.PermanentStorageException; import com.thinkaurelius.titan.diskstorage.StaticBuffer; import com.thinkaurelius.titan.diskstorage.StorageException; @@ -29,8 +28,6 @@ import org.apache.accumulo.core.client.TableNotFoundException; import org.apache.accumulo.core.client.ZooKeeperInstance; import org.apache.accumulo.core.client.admin.TableOperations; -import org.apache.accumulo.core.client.mock.MockAccumulo; -import org.apache.accumulo.core.client.mock.MockInstance; import org.apache.accumulo.core.data.Mutation; import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; @@ -51,13 +48,11 @@ public class AccumuloStoreManager extends DistributedStoreManager implements Key // Default deleter, scanner, writer parameters private static final Authorizations DEFAULT_AUTHORIZATIONS = new Authorizations(); private static final Long DEFAULT_MAX_MEMORY = 50 * 1024 * 1024l; - private static final Long DEFAULT_MAX_LATENCY = 2 * 60 * 1000l; - private static final Long DEFAULT_TIMEOUT = Long.MAX_VALUE; - private static final Integer DEFAULT_MAX_QUERY_THREADS = 3; - private static final Integer DEFAULT_MAX_WRITE_THREADS = 3; + private static final Long DEFAULT_MAX_LATENCY = 100l; + private static final Integer DEFAULT_MAX_QUERY_THREADS = 10; + private static final Integer DEFAULT_MAX_WRITE_THREADS = 10; // Configuration keys public static final String ACCUMULO_INTSANCE_KEY = "instance"; - public static final String ACCUMULO_ZOOKEEPERS_KEY = "zookeepers"; public static final String ACCUMULO_USER_KEY = "username"; public static final String ACCUMULO_PASSWORD_KEY = "password"; public static final String TABLE_NAME_KEY = "tablename"; @@ -83,14 +78,15 @@ public AccumuloStoreManager(Configuration config) throws StorageException { // Accumulo specific keys Configuration accumuloConfig = config.subset(ACCUMULO_CONFIGURATION_NAMESPACE); instanceName = accumuloConfig.getString(ACCUMULO_INTSANCE_KEY); - zooKeepers = accumuloConfig.getString(ACCUMULO_ZOOKEEPERS_KEY); + zooKeepers = accumuloConfig.getString(GraphDatabaseConfiguration.HOSTNAME_KEY, + GraphDatabaseConfiguration.HOSTNAME_DEFAULT); username = accumuloConfig.getString(ACCUMULO_USER_KEY); password = accumuloConfig.getString(ACCUMULO_PASSWORD_KEY); - instance = new ZooKeeperInstance("EtCloud", "localhost"); + instance = new ZooKeeperInstance(instanceName, zooKeepers); try { - connector = instance.getConnector("root", "bobross".getBytes()); + connector = instance.getConnector(username, password.getBytes()); } catch (AccumuloException ex) { logger.error(ex.getMessage(), ex); throw new PermanentStorageException(ex.getMessage(), ex); @@ -305,10 +301,20 @@ public void clearStorage() throws StorageException { throw new PermanentStorageException(ex); } } + + private ConcurrentMap properties; @Override public String getConfigurationProperty(final String key) throws StorageException { ensureTableExists(tableName); + + if (properties == null) { + properties = new ConcurrentHashMap(); + } + + return properties.get(key); + + /* TableOperations operations = connector.tableOperations(); try { @@ -326,11 +332,14 @@ public String getConfigurationProperty(final String key) throws StorageException logger.error(ex.getMessage(), ex); throw new PermanentStorageException(ex); } + */ } @Override public void setConfigurationProperty(final String key, final String value) throws StorageException { ensureTableExists(tableName); + + properties.put(key, value); /* TableOperations operations = connector.tableOperations(); diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java index cd76d9eb49..8e83a6f3fd 100644 --- a/titan-accumulo/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java +++ b/titan-accumulo/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java @@ -39,6 +39,7 @@ public class AccumuloStorageSetup { } Runtime.getRuntime().addShutdownHook(new Thread() { + @Override public void run() { System.out.println("All done. Shutting done Accumulo."); @@ -52,23 +53,30 @@ public void run() { } public static Configuration getAccumuloStorageConfiguration() { - BaseConfiguration config = new BaseConfiguration(); - config.addProperty(GraphDatabaseConfiguration.STORAGE_BACKEND_KEY, - "com.thinkaurelius.titan.diskstorage.accumulo.AccumuloStoreManager"); - return config; + return getAccumuloGraphConfiguration() + .subset(GraphDatabaseConfiguration.STORAGE_NAMESPACE); } public static Configuration getAccumuloGraphConfiguration() { - Configuration config = new BaseConfiguration(); + BaseConfiguration config = new BaseConfiguration(); + Configuration storageConfig = config.subset(GraphDatabaseConfiguration.STORAGE_NAMESPACE); storageConfig.addProperty(GraphDatabaseConfiguration.STORAGE_BACKEND_KEY, "com.thinkaurelius.titan.diskstorage.accumulo.AccumuloStoreManager"); + /* + storageConfig.addProperty(GraphDatabaseConfiguration.WRITE_ATTEMPTS_KEY, 10); + storageConfig.addProperty(GraphDatabaseConfiguration.READ_ATTEMPTS_KEY, 6); + storageConfig.addProperty(GraphDatabaseConfiguration.STORAGE_ATTEMPT_WAITTIME_KEY, 500); + storageConfig.addProperty(GraphDatabaseConfiguration.LOCK_RETRY_COUNT, 6); + */ + storageConfig.addProperty(GraphDatabaseConfiguration.LOCK_WAIT_MS, 1000); + Configuration accumuloConfig = storageConfig.subset(AccumuloStoreManager.ACCUMULO_CONFIGURATION_NAMESPACE); accumuloConfig.addProperty(AccumuloStoreManager.ACCUMULO_INTSANCE_KEY, "EtCloud"); - accumuloConfig.addProperty(AccumuloStoreManager.ACCUMULO_ZOOKEEPERS_KEY, "localhost"); + accumuloConfig.addProperty(GraphDatabaseConfiguration.HOSTNAME_KEY, "localhost"); accumuloConfig.addProperty(AccumuloStoreManager.ACCUMULO_USER_KEY, "root"); accumuloConfig.addProperty(AccumuloStoreManager.ACCUMULO_PASSWORD_KEY, "bobross"); diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueTest.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueTest.java index d2af385137..5e36569327 100644 --- a/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueTest.java +++ b/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueTest.java @@ -3,12 +3,13 @@ import com.thinkaurelius.titan.AccumuloStorageSetup; import com.thinkaurelius.titan.diskstorage.StorageException; import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyColumnValueStoreManager; -import org.apache.commons.configuration.Configuration; import org.junit.BeforeClass; import java.io.IOException; +import org.apache.commons.configuration.Configuration; public class AccumuloKeyColumnValueTest extends KeyColumnValueStoreTest { + @BeforeClass public static void startAccmulo() throws IOException { AccumuloStorageSetup.startAccumulo(); @@ -16,13 +17,7 @@ public static void startAccmulo() throws IOException { @Override public KeyColumnValueStoreManager openStorageManager() throws StorageException { - return new AccumuloStoreManager(getConfig()); - } - - private Configuration getConfig() { - Configuration c = AccumuloStorageSetup.getAccumuloStorageConfiguration(); - c.setProperty("accumulo-config.accumulo.zookeeper.quorum", "localhost"); - c.setProperty("accumulo-config.accumulo.zookeeper.property.clientPort", "2181"); - return c; + Configuration sc = AccumuloStorageSetup.getAccumuloStorageConfiguration(); + return new AccumuloStoreManager(sc); } } diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloMultiWriteKeyColumnValueStoreTest.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloMultiWriteKeyColumnValueStoreTest.java index c5f8b1385e..e287d60aa6 100644 --- a/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloMultiWriteKeyColumnValueStoreTest.java +++ b/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloMultiWriteKeyColumnValueStoreTest.java @@ -3,10 +3,10 @@ import com.thinkaurelius.titan.AccumuloStorageSetup; import com.thinkaurelius.titan.diskstorage.StorageException; import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyColumnValueStoreManager; -import org.apache.commons.configuration.Configuration; import org.junit.BeforeClass; import java.io.IOException; +import org.apache.commons.configuration.Configuration; public class AccumuloMultiWriteKeyColumnValueStoreTest extends MultiWriteKeyColumnValueStoreTest { @@ -17,11 +17,7 @@ public static void startAccumulo() throws IOException { @Override public KeyColumnValueStoreManager openStorageManager() throws StorageException { - return new AccumuloStoreManager(getConfig()); - } - - private Configuration getConfig() { - Configuration c = AccumuloStorageSetup.getAccumuloStorageConfiguration(); - return c; + Configuration sc = AccumuloStorageSetup.getAccumuloStorageConfiguration(); + return new AccumuloStoreManager(sc); } } diff --git a/titan-berkeleyje/pom.xml b/titan-berkeleyje/pom.xml index 041dff71fe..6cebc8ea83 100644 --- a/titan-berkeleyje/pom.xml +++ b/titan-berkeleyje/pom.xml @@ -108,10 +108,6 @@ org.apache.maven.plugins maven-surefire-report-plugin - - org.apache.maven.plugins - maven-jxr-plugin - maven-assembly-plugin 2.4 diff --git a/titan-cassandra/pom.xml b/titan-cassandra/pom.xml index 374c8061fb..426f2ed63a 100644 --- a/titan-cassandra/pom.xml +++ b/titan-cassandra/pom.xml @@ -153,10 +153,6 @@ org.apache.maven.plugins maven-surefire-report-plugin - - org.apache.maven.plugins - maven-jxr-plugin - maven-resources-plugin 2.5 diff --git a/titan-es/pom.xml b/titan-es/pom.xml index a79cce5533..0c23cf41ce 100644 --- a/titan-es/pom.xml +++ b/titan-es/pom.xml @@ -95,10 +95,6 @@ org.apache.maven.plugins maven-surefire-report-plugin - - org.apache.maven.plugins - maven-jxr-plugin - \ No newline at end of file diff --git a/titan-hbase/pom.xml b/titan-hbase/pom.xml index 77a8b572cf..392a619cd2 100644 --- a/titan-hbase/pom.xml +++ b/titan-hbase/pom.xml @@ -124,10 +124,6 @@ org.apache.maven.plugins maven-surefire-report-plugin - - org.apache.maven.plugins - maven-jxr-plugin - maven-assembly-plugin 2.4 diff --git a/titan-lucene/pom.xml b/titan-lucene/pom.xml index e544f2f373..8afd3c7877 100644 --- a/titan-lucene/pom.xml +++ b/titan-lucene/pom.xml @@ -105,10 +105,6 @@ org.apache.maven.plugins maven-surefire-report-plugin - - org.apache.maven.plugins - maven-jxr-plugin - \ No newline at end of file diff --git a/titan-test/pom.xml b/titan-test/pom.xml index afefef9685..3c20505eac 100644 --- a/titan-test/pom.xml +++ b/titan-test/pom.xml @@ -66,10 +66,6 @@ org.apache.maven.plugins maven-surefire-report-plugin - - org.apache.maven.plugins - maven-jxr-plugin - \ No newline at end of file From 7e1473a3df7310519df40021219ca9a907b1065f Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Wed, 12 Jun 2013 15:07:59 -0400 Subject: [PATCH 06/50] Removed extraneous configuration parameters from test harness. --- .../com/thinkaurelius/titan/AccumuloStorageSetup.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java index 8e83a6f3fd..a5404f19d2 100644 --- a/titan-accumulo/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java +++ b/titan-accumulo/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java @@ -64,14 +64,6 @@ public static Configuration getAccumuloGraphConfiguration() { storageConfig.addProperty(GraphDatabaseConfiguration.STORAGE_BACKEND_KEY, "com.thinkaurelius.titan.diskstorage.accumulo.AccumuloStoreManager"); - /* - storageConfig.addProperty(GraphDatabaseConfiguration.WRITE_ATTEMPTS_KEY, 10); - storageConfig.addProperty(GraphDatabaseConfiguration.READ_ATTEMPTS_KEY, 6); - storageConfig.addProperty(GraphDatabaseConfiguration.STORAGE_ATTEMPT_WAITTIME_KEY, 500); - storageConfig.addProperty(GraphDatabaseConfiguration.LOCK_RETRY_COUNT, 6); - */ - storageConfig.addProperty(GraphDatabaseConfiguration.LOCK_WAIT_MS, 1000); - Configuration accumuloConfig = storageConfig.subset(AccumuloStoreManager.ACCUMULO_CONFIGURATION_NAMESPACE); From 6b905bf2efea1c16e21a7ad11715add16cb3fd33 Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Wed, 12 Jun 2013 17:08:00 -0400 Subject: [PATCH 07/50] Minor clean up. --- titan-accumulo/pom.xml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/titan-accumulo/pom.xml b/titan-accumulo/pom.xml index 2e1cb52060..6e08870074 100644 --- a/titan-accumulo/pom.xml +++ b/titan-accumulo/pom.xml @@ -119,10 +119,6 @@ org.apache.maven.plugins maven-surefire-report-plugin - - org.apache.maven.plugins - maven-jxr-plugin - maven-assembly-plugin 2.4 From a345911741b9b528ed4b5351a615ab230117ad7f Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Thu, 13 Jun 2013 14:09:53 -0400 Subject: [PATCH 08/50] Changed Lucene dependency to 4.2.1. --- titan-accumulo/pom.xml | 7 +------ titan-lucene/pom.xml | 6 +++--- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/titan-accumulo/pom.xml b/titan-accumulo/pom.xml index 6e08870074..7447965d22 100644 --- a/titan-accumulo/pom.xml +++ b/titan-accumulo/pom.xml @@ -22,11 +22,6 @@ ${titan.version} test - - com.thinkaurelius.titan - titan-es - ${titan.version} - org.apache.accumulo accumulo-core @@ -35,7 +30,7 @@ org.apache.zookeeper zookeeper - 3.4.5 + 3.4.3 org.apache.hadoop diff --git a/titan-lucene/pom.xml b/titan-lucene/pom.xml index 8afd3c7877..67734868d7 100644 --- a/titan-lucene/pom.xml +++ b/titan-lucene/pom.xml @@ -25,17 +25,17 @@ org.apache.lucene lucene-core - 4.1.0 + 4.2.1 org.apache.lucene lucene-analyzers-common - 4.1.0 + 4.2.1 org.apache.lucene lucene-spatial - 4.1.0 + 4.2.1 From 015ed2a7f8856d29c19e7449f781e9ca3bf9afcd Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Fri, 14 Jun 2013 14:01:11 -0400 Subject: [PATCH 09/50] Fixed HOSTNAME in Accumulo back-end configuration. --- .../accumulo/AccumuloStoreManager.java | 45 +++++++++---------- .../titan/AccumuloStorageSetup.java | 6 +-- 2 files changed, 25 insertions(+), 26 deletions(-) diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java index 08e62c1660..690592ac9c 100644 --- a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java +++ b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java @@ -73,13 +73,13 @@ public class AccumuloStoreManager extends DistributedStoreManager implements Key public AccumuloStoreManager(Configuration config) throws StorageException { super(config, PORT_DEFAULT); + zooKeepers = config.getString(GraphDatabaseConfiguration.HOSTNAME_KEY, + GraphDatabaseConfiguration.HOSTNAME_DEFAULT); tableName = config.getString(TABLE_NAME_KEY, TABLE_NAME_DEFAULT); // Accumulo specific keys Configuration accumuloConfig = config.subset(ACCUMULO_CONFIGURATION_NAMESPACE); instanceName = accumuloConfig.getString(ACCUMULO_INTSANCE_KEY); - zooKeepers = accumuloConfig.getString(GraphDatabaseConfiguration.HOSTNAME_KEY, - GraphDatabaseConfiguration.HOSTNAME_DEFAULT); username = accumuloConfig.getString(ACCUMULO_USER_KEY); password = accumuloConfig.getString(ACCUMULO_PASSWORD_KEY); @@ -301,44 +301,43 @@ public void clearStorage() throws StorageException { throw new PermanentStorageException(ex); } } - private ConcurrentMap properties; @Override public String getConfigurationProperty(final String key) throws StorageException { ensureTableExists(tableName); - + if (properties == null) { properties = new ConcurrentHashMap(); } - + return properties.get(key); - + /* - TableOperations operations = connector.tableOperations(); - try { - Iterable> propIterator = operations.getProperties(tableName); + TableOperations operations = connector.tableOperations(); + try { + Iterable> propIterator = operations.getProperties(tableName); - Map properties = new HashMap(); - for (Map.Entry entry : propIterator) { - properties.put(entry.getKey(), entry.getValue()); - } - return properties.get(key); - } catch (AccumuloException ex) { - logger.error(ex.getMessage(), ex); - throw new PermanentStorageException(ex); - } catch (TableNotFoundException ex) { - logger.error(ex.getMessage(), ex); - throw new PermanentStorageException(ex); - } - */ + Map properties = new HashMap(); + for (Map.Entry entry : propIterator) { + properties.put(entry.getKey(), entry.getValue()); + } + return properties.get(key); + } catch (AccumuloException ex) { + logger.error(ex.getMessage(), ex); + throw new PermanentStorageException(ex); + } catch (TableNotFoundException ex) { + logger.error(ex.getMessage(), ex); + throw new PermanentStorageException(ex); + } + */ } @Override public void setConfigurationProperty(final String key, final String value) throws StorageException { ensureTableExists(tableName); - + properties.put(key, value); /* diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java index a5404f19d2..eec46d2493 100644 --- a/titan-accumulo/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java +++ b/titan-accumulo/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java @@ -64,13 +64,13 @@ public static Configuration getAccumuloGraphConfiguration() { storageConfig.addProperty(GraphDatabaseConfiguration.STORAGE_BACKEND_KEY, "com.thinkaurelius.titan.diskstorage.accumulo.AccumuloStoreManager"); + storageConfig.addProperty(GraphDatabaseConfiguration.HOSTNAME_KEY, "localhost"); Configuration accumuloConfig = storageConfig.subset(AccumuloStoreManager.ACCUMULO_CONFIGURATION_NAMESPACE); accumuloConfig.addProperty(AccumuloStoreManager.ACCUMULO_INTSANCE_KEY, "EtCloud"); - accumuloConfig.addProperty(GraphDatabaseConfiguration.HOSTNAME_KEY, "localhost"); - - accumuloConfig.addProperty(AccumuloStoreManager.ACCUMULO_USER_KEY, "root"); + + accumuloConfig.addProperty(AccumuloStoreManager.ACCUMULO_USER_KEY, "root"); accumuloConfig.addProperty(AccumuloStoreManager.ACCUMULO_PASSWORD_KEY, "bobross"); return config; From 1c2a2974ef1dc5fd50273db4ae0639b76fd5560c Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Fri, 14 Jun 2013 14:01:41 -0400 Subject: [PATCH 10/50] Add titan-accumulo, titan-lucene to Titan standalone. --- titan-all/pom.xml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/titan-all/pom.xml b/titan-all/pom.xml index 7ce0e9ed79..1f2332e807 100644 --- a/titan-all/pom.xml +++ b/titan-all/pom.xml @@ -26,11 +26,31 @@ titan-hbase ${titan.version} + + com.thinkaurelius.titan + titan-accumulo + ${titan.version} + + + org.apache.accumulo + accumulo-core + 1.4.3 + + + org.apache.thrift + libthrift + 0.6.1 + com.thinkaurelius.titan titan-es ${titan.version} + + com.thinkaurelius.titan + titan-lucene + ${titan.version} + ${basedir}/target From c70055f6a816c8f9e65c2596ab1e2b6859a4ac91 Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Fri, 14 Jun 2013 18:25:39 -0400 Subject: [PATCH 11/50] Use Key ranges for query slices. --- .../accumulo/AccumuloKeyColumnValueStore.java | 52 ++++++++++++------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java index 32e8f5929c..e225c5a618 100644 --- a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java +++ b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java @@ -30,7 +30,6 @@ import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.iterators.FirstEntryInRowIterator; -import org.apache.accumulo.core.iterators.user.RegExFilter; import org.apache.accumulo.core.security.Authorizations; import org.apache.hadoop.io.Text; @@ -78,14 +77,14 @@ public void close() throws StorageException { @Override public boolean containsKey(StaticBuffer key, StoreTransaction txh) throws StorageException { - Scanner scanner; + Scanner scanner; try { scanner = connector.createScanner(tableName, DEFAULT_AUTHORIZATIONS); } catch (TableNotFoundException ex) { logger.error("Should never throw this exception!", ex); throw new PermanentStorageException(ex); } - + byte[] keyBytes = key.as(StaticBuffer.ARRAY_FACTORY); scanner.setRange(new Range(new Text(keyBytes))); scanner.fetchColumnFamily(new Text(columnFamilyBytes)); @@ -104,28 +103,43 @@ public List getSlice(KeySliceQuery query, StoreTransaction txh) throws St } byte[] keyBytes = query.getKey().as(StaticBuffer.ARRAY_FACTORY); - scanner.setRange(new Range(new Text(keyBytes))); - - IteratorSetting columnfamilyIterator = new IteratorSetting(10, "cfIter", RegExFilter.class); - RegExFilter.setRegexs(columnfamilyIterator, null, columnFamily, null, null, true); - scanner.addScanIterator(columnfamilyIterator); + Text keyText = new Text(keyBytes); + Text cfText = new Text(columnFamilyBytes); byte[] startBytes = query.getSliceStart().as(StaticBuffer.ARRAY_FACTORY); byte[] endBytes = query.getSliceEnd().as(StaticBuffer.ARRAY_FACTORY); + Key startKey; + Key endKey; + + if (startBytes.length > 0) { + startKey = new Key(keyText, cfText, new Text(startBytes)); + } else { + startKey = new Key(keyText, cfText); + } + + if (endBytes.length > 0) { + endKey = new Key(keyText, cfText, new Text(endBytes)); + } else { + endKey = new Key(keyText, cfText); + } + + scanner.setRange(new Range(startKey, true, endKey, false)); + if (query.getLimit() < scanner.getBatchSize()) { + scanner.setBatchSize(query.getLimit()); + } + int count = 0; List entries = new ArrayList(); for (Map.Entry entry : scanner) { byte[] cqBytes = entry.getKey().getColumnQualifier().getBytes(); if (count < query.getLimit()) { - if (startBytes.length == 0 || Bytes.compare(startBytes, cqBytes) <= 0) { - if (endBytes.length == 0 || Bytes.compare(cqBytes, endBytes) < 0) { - entries.add(StaticBufferEntry.of(new StaticArrayBuffer(cqBytes), - new StaticArrayBuffer(entry.getValue().get()))); - count++; - } - } + entries.add(StaticBufferEntry.of(new StaticArrayBuffer(cqBytes), + new StaticArrayBuffer(entry.getValue().get()))); + count++; + } else { + break; } } @@ -251,12 +265,10 @@ public RecordIterator getKeys(StoreTransaction txh) throws Storage } scanner.setRanges(Collections.singletonList(new Range())); + + scanner.fetchColumnFamily(new Text(columnFamily)); - IteratorSetting columnfamilyIterator = new IteratorSetting(10, "cfIter", RegExFilter.class); - RegExFilter.setRegexs(columnfamilyIterator, null, this.columnFamily, null, null, true); - scanner.addScanIterator(columnfamilyIterator); - - IteratorSetting firstKeyOnlyIterator = new IteratorSetting(20, "keyIter", FirstEntryInRowIterator.class); + IteratorSetting firstKeyOnlyIterator = new IteratorSetting(10, "keyIter", FirstEntryInRowIterator.class); scanner.addScanIterator(firstKeyOnlyIterator); return new RecordIterator() { From 5bde1f264b20b571ce15c04d4e413d6f0b50a0f2 Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Sun, 16 Jun 2013 22:15:31 -0400 Subject: [PATCH 12/50] Remove HBase test resources. --- titan-accumulo/bin/get-active-master.rb | 45 --- titan-accumulo/bin/graceful_stop.sh | 104 ------ titan-accumulo/bin/hbase | 341 ------------------ titan-accumulo/bin/hbase-config.sh | 118 ------ titan-accumulo/bin/hbase-daemon.sh | 202 ----------- titan-accumulo/bin/hbase-daemons.sh | 55 --- titan-accumulo/bin/local-master-backup.sh | 55 --- titan-accumulo/bin/local-regionservers.sh | 54 --- titan-accumulo/bin/master-backup.sh | 76 ---- titan-accumulo/bin/regionservers.sh | 75 ---- titan-accumulo/bin/rolling-restart.sh | 162 --------- titan-accumulo/bin/start-hbase.sh | 54 --- titan-accumulo/bin/stop-hbase.sh | 73 ---- titan-accumulo/bin/zookeepers.sh | 61 ---- .../config/hadoop-metrics.properties | 70 ---- titan-accumulo/config/hbase-env.sh | 94 ----- titan-accumulo/config/hbase-policy.xml | 53 --- titan-accumulo/config/hbase-site.xml | 37 -- titan-accumulo/config/log4j.properties | 74 ---- titan-accumulo/config/regionservers | 1 - titan-accumulo/config/whirr-hbase.properties | 36 -- .../src/test/config/hadoop-metrics.properties | 70 ---- titan-accumulo/src/test/config/hbase-env.sh | 94 ----- .../src/test/config/hbase-policy.xml | 53 --- titan-accumulo/src/test/config/hbase-site.xml | 38 -- .../src/test/config/log4j.properties | 74 ---- titan-accumulo/src/test/config/regionservers | 1 - .../src/test/config/whirr-hbase.properties | 36 -- 28 files changed, 2206 deletions(-) delete mode 100755 titan-accumulo/bin/get-active-master.rb delete mode 100755 titan-accumulo/bin/graceful_stop.sh delete mode 100755 titan-accumulo/bin/hbase delete mode 100755 titan-accumulo/bin/hbase-config.sh delete mode 100755 titan-accumulo/bin/hbase-daemon.sh delete mode 100755 titan-accumulo/bin/hbase-daemons.sh delete mode 100755 titan-accumulo/bin/local-master-backup.sh delete mode 100755 titan-accumulo/bin/local-regionservers.sh delete mode 100755 titan-accumulo/bin/master-backup.sh delete mode 100755 titan-accumulo/bin/regionservers.sh delete mode 100755 titan-accumulo/bin/rolling-restart.sh delete mode 100755 titan-accumulo/bin/start-hbase.sh delete mode 100755 titan-accumulo/bin/stop-hbase.sh delete mode 100755 titan-accumulo/bin/zookeepers.sh delete mode 100644 titan-accumulo/config/hadoop-metrics.properties delete mode 100644 titan-accumulo/config/hbase-env.sh delete mode 100644 titan-accumulo/config/hbase-policy.xml delete mode 100644 titan-accumulo/config/hbase-site.xml delete mode 100644 titan-accumulo/config/log4j.properties delete mode 100644 titan-accumulo/config/regionservers delete mode 100644 titan-accumulo/config/whirr-hbase.properties delete mode 100644 titan-accumulo/src/test/config/hadoop-metrics.properties delete mode 100644 titan-accumulo/src/test/config/hbase-env.sh delete mode 100644 titan-accumulo/src/test/config/hbase-policy.xml delete mode 100644 titan-accumulo/src/test/config/hbase-site.xml delete mode 100644 titan-accumulo/src/test/config/log4j.properties delete mode 100644 titan-accumulo/src/test/config/regionservers delete mode 100644 titan-accumulo/src/test/config/whirr-hbase.properties diff --git a/titan-accumulo/bin/get-active-master.rb b/titan-accumulo/bin/get-active-master.rb deleted file mode 100755 index 8887a4574c..0000000000 --- a/titan-accumulo/bin/get-active-master.rb +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env hbase-jruby -# Copyright 2011 The Apache Software Foundation -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with this -# work for additional information regarding copyright ownership. The ASF -# licenses this file to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# Prints the hostname of the machine running the active master. - -include Java -import org.apache.hadoop.hbase.HBaseConfiguration -import org.apache.hadoop.hbase.ServerName -import org.apache.hadoop.hbase.zookeeper.ZKUtil -import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher - -# disable debug/info logging on this script for clarity -log_level = org.apache.log4j.Level::ERROR -org.apache.log4j.Logger.getLogger('org.apache.hadoop.hbase').setLevel(log_level) -org.apache.log4j.Logger.getLogger('org.apache.zookeeper').setLevel(log_level) - -config = HBaseConfiguration.create - -zk = ZooKeeperWatcher.new(config, 'get-active-master', nil) -begin - master_address = ZKUtil.getData(zk, zk.masterAddressZNode) - if master_address - puts ServerName.parseVersionedServerName(master_address).getHostname() - else - puts 'Master not running' - end -ensure - zk.close() -end - diff --git a/titan-accumulo/bin/graceful_stop.sh b/titan-accumulo/bin/graceful_stop.sh deleted file mode 100755 index beac4b15b1..0000000000 --- a/titan-accumulo/bin/graceful_stop.sh +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Copyright 2011 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -# Move regions off a server then stop it. Optionally restart and reload. -# Turn off the balancer before running this script. -function usage { - echo "Usage: graceful_stop.sh [--config ] [--restart [--reload]] [--thrift] [--rest] " - echo " thrift If we should stop/start thrift before/after the hbase stop/start" - echo " rest If we should stop/start rest before/after the hbase stop/start" - echo " restart If we should restart after graceful stop" - echo " reload Move offloaded regions back on to the restarted server" - echo " debug Print helpful debug information" - echo " hostname Hostname of server we are to stop" - exit 1 -} - -if [ $# -lt 1 ]; then - usage -fi - -bin=`dirname "$0"` -bin=`cd "$bin">/dev/null; pwd` -# This will set HBASE_HOME, etc. -. "$bin"/hbase-config.sh -# Get arguments -restart= -reload= -debug= -thrift= -rest= -while [ $# -gt 0 ] -do - case "$1" in - --thrift) thrift=true; shift;; - --rest) rest=true; shift;; - --restart) restart=true; shift;; - --reload) reload=true; shift;; - --debug) debug="--debug"; shift;; - --) shift; break;; - -*) usage ;; - *) break;; # terminate while loop - esac -done - -# "$@" contains the rest. Must be at least the hostname left. -if [ $# -lt 1 ]; then - usage -fi - -hostname=$1 -filename="/tmp/$hostname" -# Run the region mover script. -echo "Disabling balancer!" -echo 'balance_switch false' | "$bin"/hbase --config ${HBASE_CONF_DIR} shell -echo "Unloading $hostname region(s)" -HBASE_NOEXEC=true "$bin"/hbase --config ${HBASE_CONF_DIR} org.jruby.Main "$bin"/region_mover.rb --file=$filename $debug unload $hostname -echo "Unloaded $hostname region(s)" -# Stop the server. Have to put hostname into its own little file for hbase-daemons.sh -hosts="/tmp/$(basename $0).$$.tmp" -echo $hostname >> $hosts -if [ "$thrift" != "" ]; then - "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} stop thrift -fi -if [ "$rest" != "" ]; then - "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} stop rest -fi -"$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} stop regionserver -if [ "$restart" != "" ]; then - "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} start regionserver - if [ "$thrift" != "" ]; then - # -b 0.0.0.0 says listen on all interfaces rather than just default. - "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} start thrift -b 0.0.0.0 - fi - if [ "$rest" != "" ]; then - "$bin"/hbase-daemons.sh --config ${HBASE_CONF_DIR} --hosts ${hosts} start rest - fi - if [ "$reload" != "" ]; then - echo "Reloading $hostname region(s)" - HBASE_NOEXEC=true "$bin"/hbase --config ${HBASE_CONF_DIR} org.jruby.Main "$bin"/region_mover.rb --file=$filename $debug load $hostname - echo "Reloaded $hostname region(s)" - fi -fi - -# Cleanup tmp files. -trap "rm -f "/tmp/$(basename $0).*.tmp" &> /dev/null" EXIT diff --git a/titan-accumulo/bin/hbase b/titan-accumulo/bin/hbase deleted file mode 100755 index 62702057be..0000000000 --- a/titan-accumulo/bin/hbase +++ /dev/null @@ -1,341 +0,0 @@ -#! /usr/bin/env bash -# -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -# The hbase command script. Based on the hadoop command script putting -# in hbase classes, libs and configurations ahead of hadoop's. -# -# TODO: Narrow the amount of duplicated code. -# -# Environment Variables: -# -# JAVA_HOME The java implementation to use. Overrides JAVA_HOME. -# -# HBASE_CLASSPATH Extra Java CLASSPATH entries. -# -# HBASE_HEAPSIZE The maximum amount of heap to use, in MB. -# Default is 1000. -# -# HBASE_LIBRARY_PATH HBase additions to JAVA_LIBRARY_PATH for adding -# native libaries. -# -# HBASE_OPTS Extra Java runtime options. -# -# HBASE_CONF_DIR Alternate conf dir. Default is ${HBASE_HOME}/conf. -# -# HBASE_ROOT_LOGGER The root appender. Default is INFO,console -# -# MAVEN_HOME Where mvn is installed. -# -# JRUBY_HOME JRuby path: $JRUBY_HOME/lib/jruby.jar should exist. -# Defaults to the jar packaged with HBase. -# -# JRUBY_OPTS Extra options (eg '--1.9') passed to the hbase shell. -# Empty by default. -# -bin=`dirname "$0"` -bin=`cd "$bin">/dev/null; pwd` - -# This will set HBASE_HOME, etc. -. "$bin"/hbase-config.sh - -cygwin=false -case "`uname`" in -CYGWIN*) cygwin=true;; -esac - -# Detect if we are in hbase sources dir -in_dev_env=false -if [ -d "${HBASE_HOME}/target" ]; then - in_dev_env=true -fi - -# if no args specified, show usage -if [ $# = 0 ]; then - echo "Usage: hbase " - echo "where an option from one of these categories:" - echo "" - echo "DBA TOOLS" - echo " shell run the HBase shell" - echo " hbck run the hbase 'fsck' tool" - echo " hlog write-ahead-log analyzer" - echo " hfile store file analyzer" - echo " zkcli run the ZooKeeper shell" - echo "" - echo "PROCESS MANAGEMENT" - echo " master run an HBase HMaster node" - echo " regionserver run an HBase HRegionServer node" - echo " zookeeper run a Zookeeper server" - echo " rest run an HBase REST server" - echo " thrift run the HBase Thrift server" - echo " thrift2 run the HBase Thrift2 server" - echo " avro run an HBase Avro server" - echo "" - echo "PACKAGE MANAGEMENT" - echo " classpath dump hbase CLASSPATH" - echo " version print the version" - echo "" - echo " or" - echo " CLASSNAME run the class named CLASSNAME" - echo "Most commands print help when invoked w/o parameters." - exit 1 -fi - -# get arguments -COMMAND=$1 -shift - -JAVA=$JAVA_HOME/bin/java -JAVA_HEAP_MAX=-Xmx1000m - -MVN="mvn" -if [ "$MAVEN_HOME" != "" ]; then - MVN=${MAVEN_HOME}/bin/mvn -fi - -# override default settings for this command, if applicable -if [ -f "$HBASE_HOME/conf/hbase-env-$COMMAND.sh" ]; then - . "$HBASE_HOME/conf/hbase-env-$COMMAND.sh" -fi - -# check envvars which might override default args -if [ "$HBASE_HEAPSIZE" != "" ]; then - #echo "run with heapsize $HBASE_HEAPSIZE" - JAVA_HEAP_MAX="-Xmx""$HBASE_HEAPSIZE""m" - #echo $JAVA_HEAP_MAX -fi - -# so that filenames w/ spaces are handled correctly in loops below -IFS= - -# CLASSPATH initially contains $HBASE_CONF_DIR -CLASSPATH="${HBASE_CONF_DIR}" -CLASSPATH=${CLASSPATH}:$JAVA_HOME/lib/tools.jar - -add_maven_deps_to_classpath() { - # Need to generate classpath from maven pom. This is costly so generate it - # and cache it. Save the file into our target dir so a mvn clean will get - # clean it up and force us create a new one. - f="${HBASE_HOME}/target/cached_classpath.txt" - if [ ! -f "${f}" ] - then - ${MVN} -f "${HBASE_HOME}/pom.xml" dependency:build-classpath -Dmdep.outputFile="${f}" &> /dev/null - fi - CLASSPATH=${CLASSPATH}:`cat "${f}"` -} - -add_maven_main_classes_to_classpath() { - if [ -d "$HBASE_HOME/target/classes" ]; then - CLASSPATH=${CLASSPATH}:$HBASE_HOME/target/classes - fi -} - -add_maven_test_classes_to_classpath() { - # For developers, add hbase classes to CLASSPATH - f="$HBASE_HOME/target/test-classes" - if [ -d "${f}" ]; then - CLASSPATH=${CLASSPATH}:${f} - fi -} - -# Add maven target directory -if $in_dev_env; then - add_maven_deps_to_classpath - add_maven_main_classes_to_classpath - add_maven_test_classes_to_classpath -fi - -# For releases, add hbase & webapps to CLASSPATH -# Webapps must come first else it messes up Jetty -if [ -d "$HBASE_HOME/hbase-webapps" ]; then - CLASSPATH=${CLASSPATH}:$HBASE_HOME -fi -if [ -d "$HBASE_HOME/target/hbase-webapps" ]; then - CLASSPATH="${CLASSPATH}:${HBASE_HOME}/target" -fi -for f in $HBASE_HOME/hbase*.jar; do - if [[ $f = *sources.jar ]] - then - : # Skip sources.jar - elif [ -f $f ] - then - CLASSPATH=${CLASSPATH}:$f; - fi -done - -# Add libs to CLASSPATH -for f in $HBASE_HOME/lib/*.jar; do - CLASSPATH=${CLASSPATH}:$f; -done - -# Add user-specified CLASSPATH last -if [ "$HBASE_CLASSPATH" != "" ]; then - CLASSPATH=${CLASSPATH}:${HBASE_CLASSPATH} -fi - -# default log directory & file -if [ "$HBASE_LOG_DIR" = "" ]; then - HBASE_LOG_DIR="$HBASE_HOME/logs" -fi -if [ "$HBASE_LOGFILE" = "" ]; then - HBASE_LOGFILE='hbase.log' -fi - -# cygwin path translation -if $cygwin; then - CLASSPATH=`cygpath -p -w "$CLASSPATH"` - HBASE_HOME=`cygpath -d "$HBASE_HOME"` - HBASE_LOG_DIR=`cygpath -d "$HBASE_LOG_DIR"` -fi - -function append_path() { - if [ -z "$1" ]; then - echo $2 - else - echo $1:$2 - fi -} - -JAVA_PLATFORM="" - -#If avail, add Hadoop to the CLASSPATH and to the JAVA_LIBRARY_PATH -HADOOP_IN_PATH=$(PATH="${HADOOP_HOME:-${HADOOP_PREFIX}}/bin:$PATH" which hadoop 2>/dev/null) -if [ -f ${HADOOP_IN_PATH} ]; then - HADOOP_JAVA_LIBRARY_PATH=$(HADOOP_CLASSPATH="$CLASSPATH" ${HADOOP_IN_PATH} \ - org.apache.hadoop.hbase.util.GetJavaProperty java.library.path 2>/dev/null) - if [ -n "$HADOOP_JAVA_LIBRARY_PATH" ]; then - JAVA_LIBRARY_PATH=$(append_path "${JAVA_LIBRARY_PATH}" "$HADOOP_JAVA_LIBRARY_PATH") - fi - CLASSPATH=$(append_path "${CLASSPATH}" `${HADOOP_IN_PATH} classpath 2>/dev/null`) -fi - -if [ -d "${HBASE_HOME}/build/native" -o -d "${HBASE_HOME}/lib/native" ]; then - if [ -z $JAVA_PLATFORM ]; then - JAVA_PLATFORM=`CLASSPATH=${CLASSPATH} ${JAVA} org.apache.hadoop.util.PlatformName | sed -e "s/ /_/g"` - fi - if [ -d "$HBASE_HOME/build/native" ]; then - JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" ${HBASE_HOME}/build/native/${JAVA_PLATFORM}/lib) - fi - - if [ -d "${HBASE_HOME}/lib/native" ]; then - JAVA_LIBRARY_PATH=$(append_path "$JAVA_LIBRARY_PATH" ${HBASE_HOME}/lib/native/${JAVA_PLATFORM}) - fi -fi - -# cygwin path translation -if $cygwin; then - JAVA_LIBRARY_PATH=`cygpath -p "$JAVA_LIBRARY_PATH"` -fi - -# restore ordinary behaviour -unset IFS - -# figure out which class to run -if [ "$COMMAND" = "shell" ] ; then - # eg export JRUBY_HOME=/usr/local/share/jruby - if [ "$JRUBY_HOME" != "" ] ; then - CLASSPATH="$JRUBY_HOME/lib/jruby.jar:$CLASSPATH" - HBASE_OPTS="$HBASE_OPTS -Djruby.home=$JRUBY_HOME -Djruby.lib=$JRUBY_HOME/lib" - fi - CLASS="org.jruby.Main -X+O ${JRUBY_OPTS} ${HBASE_HOME}/bin/hirb.rb" -elif [ "$COMMAND" = "hbck" ] ; then - CLASS='org.apache.hadoop.hbase.util.HBaseFsck' -elif [ "$COMMAND" = "hlog" ] ; then - CLASS='org.apache.hadoop.hbase.regionserver.wal.HLogPrettyPrinter' -elif [ "$COMMAND" = "hfile" ] ; then - CLASS='org.apache.hadoop.hbase.io.hfile.HFile' -elif [ "$COMMAND" = "zkcli" ] ; then - # ZooKeeperMainServerArg returns '-server HOST:PORT' or empty string. - SERVER_ARG=`"$bin"/hbase org.apache.hadoop.hbase.zookeeper.ZooKeeperMainServerArg` - CLASS="org.apache.zookeeper.ZooKeeperMain ${SERVER_ARG}" - -elif [ "$COMMAND" = "master" ] ; then - CLASS='org.apache.hadoop.hbase.master.HMaster' - if [ "$1" != "stop" ] ; then - HBASE_OPTS="$HBASE_OPTS $HBASE_MASTER_OPTS" - fi -elif [ "$COMMAND" = "regionserver" ] ; then - CLASS='org.apache.hadoop.hbase.regionserver.HRegionServer' - if [ "$1" != "stop" ] ; then - HBASE_OPTS="$HBASE_OPTS $HBASE_REGIONSERVER_OPTS" - fi -elif [ "$COMMAND" = "thrift" ] ; then - CLASS='org.apache.hadoop.hbase.thrift.ThriftServer' - if [ "$1" != "stop" ] ; then - HBASE_OPTS="$HBASE_OPTS $HBASE_THRIFT_OPTS" - fi -elif [ "$COMMAND" = "thrift2" ] ; then - CLASS='org.apache.hadoop.hbase.thrift2.ThriftServer' - if [ "$1" != "stop" ] ; then - HBASE_OPTS="$HBASE_OPTS $HBASE_THRIFT_OPTS" - fi -elif [ "$COMMAND" = "rest" ] ; then - CLASS='org.apache.hadoop.hbase.rest.Main' - if [ "$1" != "stop" ] ; then - HBASE_OPTS="$HBASE_OPTS $HBASE_REST_OPTS" - fi -elif [ "$COMMAND" = "avro" ] ; then - CLASS='org.apache.hadoop.hbase.avro.AvroServer' - if [ "$1" != "stop" ] ; then - HBASE_OPTS="$HBASE_OPTS $HBASE_AVRO_OPTS" - fi -elif [ "$COMMAND" = "zookeeper" ] ; then - CLASS='org.apache.hadoop.hbase.zookeeper.HQuorumPeer' - if [ "$1" != "stop" ] ; then - HBASE_OPTS="$HBASE_OPTS $HBASE_ZOOKEEPER_OPTS" - fi - -elif [ "$COMMAND" = "classpath" ] ; then - echo $CLASSPATH - exit 0 -elif [ "$COMMAND" = "version" ] ; then - CLASS='org.apache.hadoop.hbase.util.VersionInfo' -else - CLASS=$COMMAND -fi - -# Have JVM dump heap if we run out of memory. Files will be 'launch directory' -# and are named like the following: java_pid21612.hprof. Apparently it doesn't -# 'cost' to have this flag enabled. Its a 1.6 flag only. See: -# http://blogs.sun.com/alanb/entry/outofmemoryerror_looks_a_bit_better -HBASE_OPTS="$HBASE_OPTS -Dhbase.log.dir=$HBASE_LOG_DIR" -HBASE_OPTS="$HBASE_OPTS -Dhbase.log.file=$HBASE_LOGFILE" -HBASE_OPTS="$HBASE_OPTS -Dhbase.home.dir=$HBASE_HOME" -HBASE_OPTS="$HBASE_OPTS -Dhbase.id.str=$HBASE_IDENT_STRING" -HBASE_OPTS="$HBASE_OPTS -Dhbase.root.logger=${HBASE_ROOT_LOGGER:-INFO,console}" -if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then - HBASE_OPTS="$HBASE_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH" - export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$JAVA_LIBRARY_PATH" -fi - -# Enable security logging on the master and regionserver only -if [ "$COMMAND" = "master" ] || [ "$COMMAND" = "regionserver" ]; then - HBASE_OPTS="$HBASE_OPTS -Dhbase.security.logger=${HBASE_SECURITY_LOGGER:-INFO,DRFAS}" -else - HBASE_OPTS="$HBASE_OPTS -Dhbase.security.logger=${HBASE_SECURITY_LOGGER:-INFO,NullAppender}" -fi - -# Exec unless HBASE_NOEXEC is set. -if [ "${HBASE_NOEXEC}" != "" ]; then - "$JAVA" -XX:OnOutOfMemoryError="kill -9 %p" $JAVA_HEAP_MAX $HBASE_OPTS -classpath "$CLASSPATH" $CLASS "$@" -else - exec "$JAVA" -XX:OnOutOfMemoryError="kill -9 %p" $JAVA_HEAP_MAX $HBASE_OPTS -classpath "$CLASSPATH" $CLASS "$@" -fi diff --git a/titan-accumulo/bin/hbase-config.sh b/titan-accumulo/bin/hbase-config.sh deleted file mode 100755 index 1f25713a94..0000000000 --- a/titan-accumulo/bin/hbase-config.sh +++ /dev/null @@ -1,118 +0,0 @@ -# -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -# included in all the hbase scripts with source command -# should not be executable directly -# also should not be passed any arguments, since we need original $* -# Modelled after $HADOOP_HOME/bin/hadoop-env.sh. - -# resolve links - "${BASH_SOURCE-$0}" may be a softlink - -this="${BASH_SOURCE-$0}" -while [ -h "$this" ]; do - ls=`ls -ld "$this"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '.*/.*' > /dev/null; then - this="$link" - else - this=`dirname "$this"`/"$link" - fi -done - -# convert relative path to absolute path -bin=`dirname "$this"` -script=`basename "$this"` -bin=`cd "$bin">/dev/null; pwd` -this="$bin/$script" - -# the root of the hbase installation -if [ -z "$HBASE_HOME" ]; then - export HBASE_HOME=`dirname "$this"`/.. -fi - -#check to see if the conf dir or hbase home are given as an optional arguments -while [ $# -gt 1 ] -do - if [ "--config" = "$1" ] - then - shift - confdir=$1 - shift - HBASE_CONF_DIR=$confdir - elif [ "--hosts" = "$1" ] - then - shift - hosts=$1 - shift - HBASE_REGIONSERVERS=$hosts - else - # Presume we are at end of options and break - break - fi -done - -# Allow alternate hbase conf dir location. -HBASE_CONF_DIR="${HBASE_CONF_DIR:-$HBASE_HOME/conf}" -# List of hbase regions servers. -HBASE_REGIONSERVERS="${HBASE_REGIONSERVERS:-$HBASE_CONF_DIR/regionservers}" -# List of hbase secondary masters. -HBASE_BACKUP_MASTERS="${HBASE_BACKUP_MASTERS:-$HBASE_CONF_DIR/backup-masters}" - -# Source the hbase-env.sh. Will have JAVA_HOME defined. -if [ -f "${HBASE_CONF_DIR}/hbase-env.sh" ]; then - . "${HBASE_CONF_DIR}/hbase-env.sh" -fi - -# Newer versions of glibc use an arena memory allocator that causes virtual -# memory usage to explode. Tune the variable down to prevent vmem explosion. -export MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX:-4} - -if [ -z "$JAVA_HOME" ]; then - for candidate in \ - /usr/lib/jvm/java-6-sun \ - /usr/lib/jvm/java-1.6.0-sun-1.6.0.*/jre \ - /usr/lib/jvm/java-1.6.0-sun-1.6.0.* \ - /usr/lib/j2sdk1.6-sun \ - /usr/java/jdk1.6* \ - /usr/java/jre1.6* \ - /Library/Java/Home ; do - if [ -e $candidate/bin/java ]; then - export JAVA_HOME=$candidate - break - fi - done - # if we didn't set it - if [ -z "$JAVA_HOME" ]; then - cat 1>&2 < http://java.sun.com/javase/downloads/ < | -| | -| HBase requires Java 1.6 or later. | -| NOTE: This script will find Sun Java whether you install using the | -| binary or the RPM based installer. | -+======================================================================+ -EOF - exit 1 - fi -fi diff --git a/titan-accumulo/bin/hbase-daemon.sh b/titan-accumulo/bin/hbase-daemon.sh deleted file mode 100755 index 569bbb3a38..0000000000 --- a/titan-accumulo/bin/hbase-daemon.sh +++ /dev/null @@ -1,202 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -# Runs a Hadoop hbase command as a daemon. -# -# Environment Variables -# -# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. -# HBASE_LOG_DIR Where log files are stored. PWD by default. -# HBASE_PID_DIR The pid files are stored. /tmp by default. -# HBASE_IDENT_STRING A string representing this instance of hadoop. $USER by default -# HBASE_NICENESS The scheduling priority for daemons. Defaults to 0. -# -# Modelled after $HADOOP_HOME/bin/hadoop-daemon.sh - -usage="Usage: hbase-daemon.sh [--config ]\ - (start|stop|restart) \ - " - -# if no args specified, show usage -if [ $# -le 1 ]; then - echo $usage - exit 1 -fi - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin">/dev/null; pwd` - -. "$bin"/hbase-config.sh - -# get arguments -startStop=$1 -shift - -command=$1 -shift - -hbase_rotate_log () -{ - log=$1; - num=5; - if [ -n "$2" ]; then - num=$2 - fi - if [ -f "$log" ]; then # rotate logs - while [ $num -gt 1 ]; do - prev=`expr $num - 1` - [ -f "$log.$prev" ] && mv -f "$log.$prev" "$log.$num" - num=$prev - done - mv -f "$log" "$log.$num"; - fi -} - -wait_until_done () -{ - p=$1 - cnt=${HBASE_SLAVE_TIMEOUT:-300} - origcnt=$cnt - while kill -0 $p > /dev/null 2>&1; do - if [ $cnt -gt 1 ]; then - cnt=`expr $cnt - 1` - sleep 1 - else - echo "Process did not complete after $origcnt seconds, killing." - kill -9 $p - exit 1 - fi - done - return 0 -} - -# get log directory -if [ "$HBASE_LOG_DIR" = "" ]; then - export HBASE_LOG_DIR="$HBASE_HOME/logs" -fi -mkdir -p "$HBASE_LOG_DIR" - -if [ "$HBASE_PID_DIR" = "" ]; then - HBASE_PID_DIR=/tmp -fi - -if [ "$HBASE_IDENT_STRING" = "" ]; then - export HBASE_IDENT_STRING="$USER" -fi - -# Some variables -# Work out java location so can print version into log. -if [ "$JAVA_HOME" != "" ]; then - #echo "run java in $JAVA_HOME" - JAVA_HOME=$JAVA_HOME -fi -if [ "$JAVA_HOME" = "" ]; then - echo "Error: JAVA_HOME is not set." - exit 1 -fi -JAVA=$JAVA_HOME/bin/java -export HBASE_LOG_PREFIX=hbase-$HBASE_IDENT_STRING-$command-$HOSTNAME -export HBASE_LOGFILE=$HBASE_LOG_PREFIX.log -export HBASE_ROOT_LOGGER="INFO,DRFA" -export HBASE_SECURITY_LOGGER="INFO,DRFAS" -logout=$HBASE_LOG_DIR/$HBASE_LOG_PREFIX.out -loggc=$HBASE_LOG_DIR/$HBASE_LOG_PREFIX.gc -loglog="${HBASE_LOG_DIR}/${HBASE_LOGFILE}" -pid=$HBASE_PID_DIR/hbase-$HBASE_IDENT_STRING-$command.pid - -if [ "$HBASE_USE_GC_LOGFILE" = "true" ]; then - export HBASE_GC_OPTS=" -Xloggc:${loggc}" -fi - -# Set default scheduling priority -if [ "$HBASE_NICENESS" = "" ]; then - export HBASE_NICENESS=0 -fi - -case $startStop in - - (start) - mkdir -p "$HBASE_PID_DIR" - if [ -f $pid ]; then - if kill -0 `cat $pid` > /dev/null 2>&1; then - echo $command running as process `cat $pid`. Stop it first. - exit 1 - fi - fi - - hbase_rotate_log $logout - hbase_rotate_log $loggc - echo starting $command, logging to $logout - # Add to the command log file vital stats on our environment. - echo "`date` Starting $command on `hostname`" >> $loglog - echo "`ulimit -a`" >> $loglog 2>&1 - nohup nice -n $HBASE_NICENESS "$HBASE_HOME"/bin/hbase \ - --config "${HBASE_CONF_DIR}" \ - $command "$@" $startStop > "$logout" 2>&1 < /dev/null & - echo $! > $pid - sleep 1; head "$logout" - ;; - - (stop) - if [ -f $pid ]; then - # kill -0 == see if the PID exists - if kill -0 `cat $pid` > /dev/null 2>&1; then - echo -n stopping $command - echo "`date` Terminating $command" >> $loglog - kill `cat $pid` > /dev/null 2>&1 - while kill -0 `cat $pid` > /dev/null 2>&1; do - echo -n "." - sleep 1; - done - rm $pid - echo - else - retval=$? - echo no $command to stop because kill -0 of pid `cat $pid` failed with status $retval - fi - else - echo no $command to stop because no pid file $pid - fi - ;; - - (restart) - thiscmd=$0 - args=$@ - # stop the command - $thiscmd --config "${HBASE_CONF_DIR}" stop $command $args & - wait_until_done $! - # wait a user-specified sleep period - sp=${HBASE_RESTART_SLEEP:-3} - if [ $sp -gt 0 ]; then - sleep $sp - fi - # start the command - $thiscmd --config "${HBASE_CONF_DIR}" start $command $args & - wait_until_done $! - ;; - - (*) - echo $usage - exit 1 - ;; - -esac diff --git a/titan-accumulo/bin/hbase-daemons.sh b/titan-accumulo/bin/hbase-daemons.sh deleted file mode 100755 index 843eaaa74f..0000000000 --- a/titan-accumulo/bin/hbase-daemons.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -# Run a hbase command on all slave hosts. -# Modelled after $HADOOP_HOME/bin/hadoop-daemons.sh - -usage="Usage: hbase-daemons.sh [--config ] \ - [--hosts regionserversfile] [start|stop] command args..." - -# if no args specified, show usage -if [ $# -le 1 ]; then - echo $usage - exit 1 -fi - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin">/dev/null; pwd` - -. $bin/hbase-config.sh - -remote_cmd="cd ${HBASE_HOME}; $bin/hbase-daemon.sh --config ${HBASE_CONF_DIR} $@" -args="--hosts ${HBASE_REGIONSERVERS} --config ${HBASE_CONF_DIR} $remote_cmd" - -command=$2 -case $command in - (zookeeper) - exec "$bin/zookeepers.sh" $args - ;; - (master-backup) - exec "$bin/master-backup.sh" $args - ;; - (*) - exec "$bin/regionservers.sh" $args - ;; -esac - diff --git a/titan-accumulo/bin/local-master-backup.sh b/titan-accumulo/bin/local-master-backup.sh deleted file mode 100755 index 2c0a4c02c7..0000000000 --- a/titan-accumulo/bin/local-master-backup.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/sh -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# This is used for starting multiple masters on the same machine. -# run it from hbase-dir/ just like 'bin/hbase' -# Supports up to 10 masters (limitation = overlapping ports) - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin" >/dev/null && pwd` - -if [ $# -lt 2 ]; then - S=`basename "${BASH_SOURCE-$0}"` - echo "Usage: $S [start|stop] offset(s)" - echo "" - echo " e.g. $S start 1" - exit -fi - -# sanity check: make sure your master opts don't use ports [i.e. JMX/DBG] -export HBASE_MASTER_OPTS=" " - -run_master () { - DN=$2 - export HBASE_IDENT_STRING="$USER-$DN" - HBASE_MASTER_ARGS="\ - -D hbase.master.port=`expr 60000 + $DN` \ - -D hbase.master.info.port=`expr 60010 + $DN` \ - --backup" - "$bin"/hbase-daemon.sh $1 master $HBASE_MASTER_ARGS -} - -cmd=$1 -shift; - -for i in $* -do - run_master $cmd $i -done diff --git a/titan-accumulo/bin/local-regionservers.sh b/titan-accumulo/bin/local-regionservers.sh deleted file mode 100755 index a4d5a1d932..0000000000 --- a/titan-accumulo/bin/local-regionservers.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/sh -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# This is used for starting multiple regionservers on the same machine. -# run it from hbase-dir/ just like 'bin/hbase' -# Supports up to 100 regionservers (limitation = overlapping ports) - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin" >/dev/null && pwd` - -if [ $# -lt 2 ]; then - S=`basename "${BASH_SOURCE-$0}"` - echo "Usage: $S [start|stop] offset(s)" - echo "" - echo " e.g. $S start 1 2" - exit -fi - -# sanity check: make sure your regionserver opts don't use ports [i.e. JMX/DBG] -export HBASE_REGIONSERVER_OPTS=" " - -run_regionserver () { - DN=$2 - export HBASE_IDENT_STRING="$USER-$DN" - HBASE_REGIONSERVER_ARGS="\ - -D hbase.regionserver.port=`expr 60200 + $DN` \ - -D hbase.regionserver.info.port=`expr 60300 + $DN`" - "$bin"/hbase-daemon.sh $1 regionserver $HBASE_REGIONSERVER_ARGS -} - -cmd=$1 -shift; - -for i in $* -do - run_regionserver $cmd $i -done diff --git a/titan-accumulo/bin/master-backup.sh b/titan-accumulo/bin/master-backup.sh deleted file mode 100755 index 114757c763..0000000000 --- a/titan-accumulo/bin/master-backup.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Copyright 2010 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -# Run a shell command on all backup master hosts. -# -# Environment Variables -# -# HBASE_BACKUP_MASTERS File naming remote hosts. -# Default is ${HBASE_CONF_DIR}/backup-masters -# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. -# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. -# HADOOP_SLAVE_SLEEP Seconds to sleep between spawning remote commands. -# HADOOP_SSH_OPTS Options passed to ssh when running remote commands. -# -# Modelled after $HADOOP_HOME/bin/slaves.sh. - -usage="Usage: $0 [--config ] command..." - -# if no args specified, show usage -if [ $# -le 0 ]; then - echo $usage - exit 1 -fi - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin">/dev/null; pwd` - -. "$bin"/hbase-config.sh - -# If the master backup file is specified in the command line, -# then it takes precedence over the definition in -# hbase-env.sh. Save it here. -HOSTLIST=$HBASE_BACKUP_MASTERS - -if [ "$HOSTLIST" = "" ]; then - if [ "$HBASE_BACKUP_MASTERS" = "" ]; then - export HOSTLIST="${HBASE_CONF_DIR}/backup-masters" - else - export HOSTLIST="${HBASE_BACKUP_MASTERS}" - fi -fi - - -args=${@// /\\ } -args=${args/master-backup/master} - -if [ -f $HOSTLIST ]; then - for hmaster in `cat "$HOSTLIST"`; do - ssh $HBASE_SSH_OPTS $hmaster $"$args --backup" \ - 2>&1 | sed "s/^/$hmaster: /" & - if [ "$HBASE_SLAVE_SLEEP" != "" ]; then - sleep $HBASE_SLAVE_SLEEP - fi - done -fi - -wait diff --git a/titan-accumulo/bin/regionservers.sh b/titan-accumulo/bin/regionservers.sh deleted file mode 100755 index 9759f2b00c..0000000000 --- a/titan-accumulo/bin/regionservers.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -# Run a shell command on all regionserver hosts. -# -# Environment Variables -# -# HBASE_REGIONSERVERS File naming remote hosts. -# Default is ${HADOOP_CONF_DIR}/regionservers -# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. -# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. -# HADOOP_SLAVE_SLEEP Seconds to sleep between spawning remote commands. -# HADOOP_SSH_OPTS Options passed to ssh when running remote commands. -# -# Modelled after $HADOOP_HOME/bin/slaves.sh. - -usage="Usage: regionservers [--config ] command..." - -# if no args specified, show usage -if [ $# -le 0 ]; then - echo $usage - exit 1 -fi - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin">/dev/null; pwd` - -. "$bin"/hbase-config.sh - -# If the regionservers file is specified in the command line, -# then it takes precedence over the definition in -# hbase-env.sh. Save it here. -HOSTLIST=$HBASE_REGIONSERVERS - -if [ "$HOSTLIST" = "" ]; then - if [ "$HBASE_REGIONSERVERS" = "" ]; then - export HOSTLIST="${HBASE_CONF_DIR}/regionservers" - else - export HOSTLIST="${HBASE_REGIONSERVERS}" - fi -fi - -for regionserver in `cat "$HOSTLIST"`; do - if ${HBASE_SLAVE_PARALLEL:-true}; then - ssh $HBASE_SSH_OPTS $regionserver $"${@// /\\ }" \ - 2>&1 | sed "s/^/$regionserver: /" & - else # run each command serially - ssh $HBASE_SSH_OPTS $regionserver $"${@// /\\ }" \ - 2>&1 | sed "s/^/$regionserver: /" - fi - if [ "$HBASE_SLAVE_SLEEP" != "" ]; then - sleep $HBASE_SLAVE_SLEEP - fi -done - -wait diff --git a/titan-accumulo/bin/rolling-restart.sh b/titan-accumulo/bin/rolling-restart.sh deleted file mode 100755 index 07a78c71ed..0000000000 --- a/titan-accumulo/bin/rolling-restart.sh +++ /dev/null @@ -1,162 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -# Run a shell command on all regionserver hosts. -# -# Environment Variables -# -# HBASE_REGIONSERVERS File naming remote hosts. -# Default is ${HADOOP_CONF_DIR}/regionservers -# HADOOP_CONF_DIR Alternate conf dir. Default is ${HADOOP_HOME}/conf. -# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. -# HADOOP_SLAVE_SLEEP Seconds to sleep between spawning remote commands. -# HADOOP_SLAVE_TIMEOUT Seconds to wait for timing out a remote command. -# HADOOP_SSH_OPTS Options passed to ssh when running remote commands. -# -# Modelled after $HADOOP_HOME/bin/slaves.sh. - -usage="Usage: $0 [--config ] [--rs-only] [--master-only] [--graceful]" - -bin=`dirname "$0"` -bin=`cd "$bin">/dev/null; pwd` - -. "$bin"/hbase-config.sh - -# start hbase daemons -errCode=$? -if [ $errCode -ne 0 ] -then - exit $errCode -fi - -function usage() { - echo $usage - exit 1 -} - - -RR_RS=1 -RR_MASTER=1 -RR_GRACEFUL=0 - -for x in "$@" ; do - case "$x" in - --rs-only|-r) - RR_RS=1 - RR_MASTER=0 - RR_GRACEFUL=0 - ;; - --master-only) - RR_RS=0 - RR_MASTER=1 - RR_GRACEFUL=0 - ;; - --graceful) - RR_RS=0 - RR_MASTER=0 - RR_GRACEFUL=1 - ;; - *) - echo Bad argument: $x - usage - exit 1 - ;; - esac -done - -# quick function to get a value from the HBase config file -# HBASE-6504 - only take the first line of the output in case verbose gc is on -distMode=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed | head -n 1` -if [ "$distMode" == 'false' ]; then - if [ $RR_RS -ne 1 ] || [ $RR_MASTER -ne 1 ]; then - echo Cant do selective rolling restart if not running distributed - exit 1 - fi - "$bin"/hbase-daemon.sh restart master -else - zparent=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.parent` - if [ "$zparent" == "null" ]; then zparent="/hbase"; fi - - if [ $RR_MASTER -eq 1 ]; then - # stop all masters before re-start to avoid races for master znode - "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" stop master - "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ - --hosts "${HBASE_BACKUP_MASTERS}" stop master-backup - - # make sure the master znode has been deleted before continuing - zmaster=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.master` - if [ "$zmaster" == "null" ]; then zmaster="master"; fi - zmaster=$zparent/$zmaster - echo -n "Waiting for Master ZNode ${zmaster} to expire" - while ! bin/hbase zkcli stat $zmaster 2>&1 | grep "Node does not exist"; do - echo -n "." - sleep 1 - done - echo #force a newline - - # all masters are down, now restart - "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" start master - "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ - --hosts "${HBASE_BACKUP_MASTERS}" start master-backup - - echo "Wait a minute for master to come up join cluster" - sleep 60 - - # Master joing cluster will start in cleaning out regions in transition. - # Wait until the master has cleaned out regions in transition before - # giving it a bunch of work to do; master is vulnerable during startup - zunassigned=`$bin/hbase org.apache.hadoop.hbase.util.HBaseConfTool zookeeper.znode.unassigned` - if [ "$zunassigned" == "null" ]; then zunassigned="unassigned"; fi - zunassigned="$zparent/$zunassigned" - echo -n "Waiting for ${zunassigned} to empty" - while true ; do - unassigned=`$bin/hbase zkcli stat ${zunassigned} 2>&1 |grep -e 'numChildren = '|sed -e 's,numChildren = ,,'` - if test 0 -eq ${unassigned} - then - break - else - echo -n " ${unassigned}" - fi - sleep 1 - done - fi - - if [ $RR_RS -eq 1 ]; then - # unlike the masters, roll all regionservers one-at-a-time - export HBASE_SLAVE_PARALLEL=false - "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ - --hosts "${HBASE_REGIONSERVERS}" restart regionserver - fi - - if [ $RR_GRACEFUL -eq 1 ]; then - # gracefully restart all online regionservers - online_regionservers=`$bin/hbase zkcli ls $zparent/rs 2>&1 | tail -1 | sed "s/\[//" | sed "s/\]//"` - for rs in $online_regionservers - do - rs_parts=(${rs//,/ }) - hostname=${rs_parts[0]} - echo "Gracefully restarting: $hostname" - "$bin"/graceful_stop.sh --config "${HBASE_CONF_DIR}" --restart --reload --debug "$hostname" - sleep 1 - done - fi -fi diff --git a/titan-accumulo/bin/start-hbase.sh b/titan-accumulo/bin/start-hbase.sh deleted file mode 100755 index 0b0ce1a58d..0000000000 --- a/titan-accumulo/bin/start-hbase.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -# Modelled after $HADOOP_HOME/bin/start-hbase.sh. - -# Start hadoop hbase daemons. -# Run this on master node. -usage="Usage: start-hbase.sh" - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin">/dev/null; pwd` - -. "$bin"/hbase-config.sh - -# start hbase daemons -errCode=$? -if [ $errCode -ne 0 ] -then - exit $errCode -fi -# HBASE-6504 - only take the first line of the output in case verbose gc is on -distMode=`$bin/hbase --config "$HBASE_CONF_DIR" org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed | head -n 1` - - -if [ "$distMode" == 'false' ] -then - "$bin"/hbase-daemon.sh start master -else - "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" start zookeeper - "$bin"/hbase-daemon.sh --config "${HBASE_CONF_DIR}" start master - "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ - --hosts "${HBASE_REGIONSERVERS}" start regionserver - "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ - --hosts "${HBASE_BACKUP_MASTERS}" start master-backup -fi diff --git a/titan-accumulo/bin/stop-hbase.sh b/titan-accumulo/bin/stop-hbase.sh deleted file mode 100755 index e9749fee28..0000000000 --- a/titan-accumulo/bin/stop-hbase.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -# Modelled after $HADOOP_HOME/bin/stop-hbase.sh. - -# Stop hadoop hbase daemons. Run this on master node. - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin">/dev/null; pwd` - -. "$bin"/hbase-config.sh - -# variables needed for stop command -if [ "$HBASE_LOG_DIR" = "" ]; then - export HBASE_LOG_DIR="$HBASE_HOME/logs" -fi -mkdir -p "$HBASE_LOG_DIR" - -if [ "$HBASE_IDENT_STRING" = "" ]; then - export HBASE_IDENT_STRING="$USER" -fi - -export HBASE_LOG_PREFIX=hbase-$HBASE_IDENT_STRING-master-$HOSTNAME -export HBASE_LOGFILE=$HBASE_LOG_PREFIX.log -logout=$HBASE_LOG_DIR/$HBASE_LOG_PREFIX.out -loglog="${HBASE_LOG_DIR}/${HBASE_LOGFILE}" -pid=${HBASE_PID_DIR:-/tmp}/hbase-$HBASE_IDENT_STRING-master.pid - -echo -n stopping hbase -echo "`date` Stopping hbase (via master)" >> $loglog - -nohup nice -n ${HBASE_NICENESS:-0} "$HBASE_HOME"/bin/hbase \ - --config "${HBASE_CONF_DIR}" \ - master stop "$@" > "$logout" 2>&1 < /dev/null & - -while kill -0 `cat $pid` > /dev/null 2>&1; do - echo -n "." - sleep 1; -done -# Add a CR after we're done w/ dots. -echo - -# distributed == false means that the HMaster will kill ZK when it exits -# HBASE-6504 - only take the first line of the output in case verbose gc is on -distMode=`$bin/hbase --config "$HBASE_CONF_DIR" org.apache.hadoop.hbase.util.HBaseConfTool hbase.cluster.distributed | head -n 1` -if [ "$distMode" == 'true' ] -then - # TODO: store backup masters in ZooKeeper and have the primary send them a shutdown message - # stop any backup masters - "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" \ - --hosts "${HBASE_BACKUP_MASTERS}" stop master-backup - - "$bin"/hbase-daemons.sh --config "${HBASE_CONF_DIR}" stop zookeeper -fi diff --git a/titan-accumulo/bin/zookeepers.sh b/titan-accumulo/bin/zookeepers.sh deleted file mode 100755 index 89a214e5a8..0000000000 --- a/titan-accumulo/bin/zookeepers.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env bash -# -#/** -# * Copyright 2009 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ -# -# Run a shell command on all zookeeper hosts. -# -# Environment Variables -# -# HBASE_CONF_DIR Alternate hbase conf dir. Default is ${HBASE_HOME}/conf. -# HBASE_SLAVE_SLEEP Seconds to sleep between spawning remote commands. -# HBASE_SSH_OPTS Options passed to ssh when running remote commands. -# -# Modelled after $HADOOP_HOME/bin/slaves.sh. - -usage="Usage: zookeepers [--config ] command..." - -# if no args specified, show usage -if [ $# -le 0 ]; then - echo $usage - exit 1 -fi - -bin=`dirname "${BASH_SOURCE-$0}"` -bin=`cd "$bin">/dev/null; pwd` - -. "$bin"/hbase-config.sh - -if [ "$HBASE_MANAGES_ZK" = "" ]; then - HBASE_MANAGES_ZK=true -fi - -if [ "$HBASE_MANAGES_ZK" = "true" ]; then - hosts=`"$bin"/hbase org.apache.hadoop.hbase.zookeeper.ZKServerTool | grep '^ZK host:' | sed 's,^ZK host:,,'` - cmd=$"${@// /\\ }" - for zookeeper in $hosts; do - ssh $HBASE_SSH_OPTS $zookeeper $cmd 2>&1 | sed "s/^/$zookeeper: /" & - if [ "$HBASE_SLAVE_SLEEP" != "" ]; then - sleep $HBASE_SLAVE_SLEEP - fi - done -fi - -wait diff --git a/titan-accumulo/config/hadoop-metrics.properties b/titan-accumulo/config/hadoop-metrics.properties deleted file mode 100644 index 2060e22df5..0000000000 --- a/titan-accumulo/config/hadoop-metrics.properties +++ /dev/null @@ -1,70 +0,0 @@ -# See http://wiki.apache.org/hadoop/GangliaMetrics -# Make sure you know whether you are using ganglia 3.0 or 3.1. -# If 3.1, you will have to patch your hadoop instance with HADOOP-4675 -# And, yes, this file is named hadoop-metrics.properties rather than -# hbase-metrics.properties because we're leveraging the hadoop metrics -# package and hadoop-metrics.properties is an hardcoded-name, at least -# for the moment. -# -# See also http://hadoop.apache.org/hbase/docs/current/metrics.html -# GMETADHOST_IP is the hostname (or) IP address of the server on which the ganglia -# meta daemon (gmetad) service is running - -# Configuration of the "hbase" context for NullContextWithUpdateThread -# NullContextWithUpdateThread is a null context which has a thread calling -# periodically when monitoring is started. This keeps the data sampled -# correctly. -hbase.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread -hbase.period=10 - -# Configuration of the "hbase" context for file -# hbase.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext -# hbase.fileName=/tmp/metrics_hbase.log - -# HBase-specific configuration to reset long-running stats (e.g. compactions) -# If this variable is left out, then the default is no expiration. -hbase.extendedperiod = 3600 - -# Configuration of the "hbase" context for ganglia -# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) -# hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext -# hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 -# hbase.period=10 -# hbase.servers=GMETADHOST_IP:8649 - -# Configuration of the "jvm" context for null -jvm.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread -jvm.period=10 - -# Configuration of the "jvm" context for file -# jvm.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext -# jvm.fileName=/tmp/metrics_jvm.log - -# Configuration of the "jvm" context for ganglia -# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) -# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext -# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 -# jvm.period=10 -# jvm.servers=GMETADHOST_IP:8649 - -# Configuration of the "rpc" context for null -rpc.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread -rpc.period=10 - -# Configuration of the "rpc" context for file -# rpc.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext -# rpc.fileName=/tmp/metrics_rpc.log - -# Configuration of the "rpc" context for ganglia -# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) -# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext -# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 -# rpc.period=10 -# rpc.servers=GMETADHOST_IP:8649 - -# Configuration of the "rest" context for ganglia -# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) -# rest.class=org.apache.hadoop.metrics.ganglia.GangliaContext -# rest.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 -# rest.period=10 -# rest.servers=GMETADHOST_IP:8649 diff --git a/titan-accumulo/config/hbase-env.sh b/titan-accumulo/config/hbase-env.sh deleted file mode 100644 index 4590184793..0000000000 --- a/titan-accumulo/config/hbase-env.sh +++ /dev/null @@ -1,94 +0,0 @@ -# -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -# Set environment variables here. - -# The java implementation to use. Java 1.6 required. -# export JAVA_HOME=/usr/java/jdk1.6.0/ - -# Extra Java CLASSPATH elements. Optional. -export HBASE_CLASSPATH=$HBASE_CLASSPATH:./target/test-lib - -# The maximum amount of heap to use, in MB. Default is 1000. -# export HBASE_HEAPSIZE=1000 - -# Extra Java runtime options. -# Below are what we set by default. May only work with SUN JVM. -# For more on why as well as other possible settings, -# see http://wiki.apache.org/hadoop/PerformanceTuning -export HBASE_OPTS="$HBASE_OPTS -XX:+UseConcMarkSweepGC" - -# Uncomment below to enable java garbage collection logging in the .out file. -# export HBASE_OPTS="$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps $HBASE_GC_OPTS" - -# Uncomment below (along with above GC logging) to put GC information in its own logfile (will set HBASE_GC_OPTS) -# export HBASE_USE_GC_LOGFILE=true - - -# Uncomment below if you intend to use the EXPERIMENTAL off heap cache. -# export HBASE_OPTS="$HBASE_OPTS -XX:MaxDirectMemorySize=" -# Set hbase.offheapcache.percentage in hbase-site.xml to a nonzero value. - - -# Uncomment and adjust to enable JMX exporting -# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access. -# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html -# -# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" -# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10101" -# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10102" -# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103" -# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104" - -# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default. -# export HBASE_REGIONSERVERS=${HBASE_HOME}/conf/regionservers - -# File naming hosts on which backup HMaster will run. $HBASE_HOME/conf/backup-masters by default. -# export HBASE_BACKUP_MASTERS=${HBASE_HOME}/conf/backup-masters - -# Extra ssh options. Empty by default. -# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR" - -# Where log files are stored. $HBASE_HOME/logs by default. -# export HBASE_LOG_DIR=${HBASE_HOME}/logs - -# Enable remote JDWP debugging of major HBase processes. Meant for Core Developers -# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8070" -# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8071" -# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8072" -# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8073" - -# A string representing this instance of hbase. $USER by default. -# export HBASE_IDENT_STRING=$USER - -# The scheduling priority for daemon processes. See 'man nice'. -# export HBASE_NICENESS=10 - -# The directory where pid files are stored. /tmp by default. -# export HBASE_PID_DIR=/var/hadoop/pids - -# Seconds to sleep between slave commands. Unset by default. This -# can be useful in large clusters, where, e.g., slave rsyncs can -# otherwise arrive faster than the master can service them. -# export HBASE_SLAVE_SLEEP=0.1 - -# Tell HBase whether it should manage it's own instance of Zookeeper or not. -# export HBASE_MANAGES_ZK=true diff --git a/titan-accumulo/config/hbase-policy.xml b/titan-accumulo/config/hbase-policy.xml deleted file mode 100644 index e45f23c962..0000000000 --- a/titan-accumulo/config/hbase-policy.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - - - - - security.client.protocol.acl - * - ACL for HRegionInterface protocol implementations (ie. - clients talking to HRegionServers) - The ACL is a comma-separated list of user and group names. The user and - group list is separated by a blank. For e.g. "alice,bob users,wheel". - A special value of "*" means all users are allowed. - - - - security.admin.protocol.acl - * - ACL for HMasterInterface protocol implementation (ie. - clients talking to HMaster for admin operations). - The ACL is a comma-separated list of user and group names. The user and - group list is separated by a blank. For e.g. "alice,bob users,wheel". - A special value of "*" means all users are allowed. - - - - security.masterregion.protocol.acl - * - ACL for HMasterRegionInterface protocol implementations - (for HRegionServers communicating with HMaster) - The ACL is a comma-separated list of user and group names. The user and - group list is separated by a blank. For e.g. "alice,bob users,wheel". - A special value of "*" means all users are allowed. - - diff --git a/titan-accumulo/config/hbase-site.xml b/titan-accumulo/config/hbase-site.xml deleted file mode 100644 index a0587829a1..0000000000 --- a/titan-accumulo/config/hbase-site.xml +++ /dev/null @@ -1,37 +0,0 @@ - - - - - - hbase.rootdir - file:///tmp/hbase - - - hbase.zookeeper.property.dataDir - /tmp/zookeeper - - - diff --git a/titan-accumulo/config/log4j.properties b/titan-accumulo/config/log4j.properties deleted file mode 100644 index f6c3305178..0000000000 --- a/titan-accumulo/config/log4j.properties +++ /dev/null @@ -1,74 +0,0 @@ -# Define some default values that can be overridden by system properties -hbase.root.logger=INFO,console -hbase.security.logger=INFO,console -hbase.log.dir=. -hbase.log.file=hbase.log - -# Define the root logger to the system property "hbase.root.logger". -log4j.rootLogger=${hbase.root.logger} - -# Logging Threshold -log4j.threshold=ALL - -# -# Daily Rolling File Appender -# -log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender -log4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file} - -# Rollver at midnight -log4j.appender.DRFA.DatePattern=.yyyy-MM-dd - -# 30-day backup -#log4j.appender.DRFA.MaxBackupIndex=30 -log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout - -# Pattern format: Date LogLevel LoggerName LogMessage -log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n - -# Debugging Pattern format -#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n - -# -# Security audit appender -# -hbase.security.log.file=SecurityAuth.audit -log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender -log4j.appender.DRFAS.File=${hbase.log.dir}/${hbase.security.log.file} -log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout -log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n -log4j.category.SecurityLogger=${hbase.security.logger} -log4j.additivity.SecurityLogger=false - -# -# Null Appender -# -log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender - -# -# console -# Add "console" to rootlogger above if you want to use this -# -log4j.appender.console=org.apache.log4j.ConsoleAppender -log4j.appender.console.target=System.err -log4j.appender.console.layout=org.apache.log4j.PatternLayout -log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n - -# Custom Logging levels - -log4j.logger.org.apache.zookeeper=INFO -#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG -log4j.logger.org.apache.hadoop.hbase=DEBUG -# Make these two classes INFO-level. Make them DEBUG to see more zk debug. -log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=INFO -log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher=INFO -#log4j.logger.org.apache.hadoop.dfs=DEBUG -# Set this class to log INFO only otherwise its OTT - -# Uncomment this line to enable tracing on _every_ RPC call (this can be a lot of output) -#log4j.logger.org.apache.hadoop.ipc.HBaseServer.trace=DEBUG - -# Uncomment the below if you want to remove logging of client region caching' -# and scan of .META. messages -# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=INFO -# log4j.logger.org.apache.hadoop.hbase.client.MetaScanner=INFO diff --git a/titan-accumulo/config/regionservers b/titan-accumulo/config/regionservers deleted file mode 100644 index 2fbb50c4a8..0000000000 --- a/titan-accumulo/config/regionservers +++ /dev/null @@ -1 +0,0 @@ -localhost diff --git a/titan-accumulo/config/whirr-hbase.properties b/titan-accumulo/config/whirr-hbase.properties deleted file mode 100644 index a9ced05fb7..0000000000 --- a/titan-accumulo/config/whirr-hbase.properties +++ /dev/null @@ -1,36 +0,0 @@ -# A Whirr Receipe for HBase + Hadoop -# Read the Configuration Guide for more info: -# http://whirr.apache.org/docs/latest/configuration-guide.html - -# Change the cluster name here -whirr.cluster-name=hbase-hadoop - -# Change the number of machines in the cluster here -whirr.instance-templates=1 zookeeper+hadoop-namenode+hadoop-jobtracker+hbase-master,3 hadoop-datanode+hadoop-tasktracker+hbase-regionserver - -# replication level should not be higher than number of data nodes -hbase-site.dfs.replication=2 - -# For EC2 set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables. -whirr.provider=aws-ec2 -whirr.identity=${env:AWS_ACCESS_KEY_ID} -whirr.credential=${env:AWS_SECRET_ACCESS_KEY} - -# The size of the instance to use. See http://aws.amazon.com/ec2/instance-types/ -whirr.hardware-id=m1.large -# Ubuntu 10.04 LTS Lucid. See http://alestic.com/ -whirr.image-id=us-east-1/ami-da0cf8b3 -# If you choose a different location, make sure whirr.image-id is updated too -whirr.location-id=us-east-1 - -# By default use the user system SSH keys. Override them here. -# whirr.private-key-file=${sys:user.home}/.ssh/id_rsa -# whirr.public-key-file=${whirr.private-key-file}.pub - -whirr.hbase.tarball.url=http://archive.apache.org/dist/hbase/hbase-0.94.1/hbase-0.94.1.tar.gz -whirr.hadoop.tarball.url=http://archive.apache.org/dist/hadoop/core/hadoop-1.0.3/hadoop-1.0.3.tar.gz - -# Options for the hbase master & regionserver processes -#hbase-env.HBASE_MASTER_OPTS=-Xms1000m -Xmx1000m -Xmn256m -XX:+UseConcMarkSweepGC -XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -Xloggc:/data/hbase/logs/hbase-master-gc.log -#hbase-env.HBASE_REGIONSERVER_OPTS=-Xms2000m -Xmx2000m -Xmn256m -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=88 -XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -Xloggc:/data/hbase/logs/hbase-regionserver-gc.log - diff --git a/titan-accumulo/src/test/config/hadoop-metrics.properties b/titan-accumulo/src/test/config/hadoop-metrics.properties deleted file mode 100644 index 2060e22df5..0000000000 --- a/titan-accumulo/src/test/config/hadoop-metrics.properties +++ /dev/null @@ -1,70 +0,0 @@ -# See http://wiki.apache.org/hadoop/GangliaMetrics -# Make sure you know whether you are using ganglia 3.0 or 3.1. -# If 3.1, you will have to patch your hadoop instance with HADOOP-4675 -# And, yes, this file is named hadoop-metrics.properties rather than -# hbase-metrics.properties because we're leveraging the hadoop metrics -# package and hadoop-metrics.properties is an hardcoded-name, at least -# for the moment. -# -# See also http://hadoop.apache.org/hbase/docs/current/metrics.html -# GMETADHOST_IP is the hostname (or) IP address of the server on which the ganglia -# meta daemon (gmetad) service is running - -# Configuration of the "hbase" context for NullContextWithUpdateThread -# NullContextWithUpdateThread is a null context which has a thread calling -# periodically when monitoring is started. This keeps the data sampled -# correctly. -hbase.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread -hbase.period=10 - -# Configuration of the "hbase" context for file -# hbase.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext -# hbase.fileName=/tmp/metrics_hbase.log - -# HBase-specific configuration to reset long-running stats (e.g. compactions) -# If this variable is left out, then the default is no expiration. -hbase.extendedperiod = 3600 - -# Configuration of the "hbase" context for ganglia -# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) -# hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext -# hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 -# hbase.period=10 -# hbase.servers=GMETADHOST_IP:8649 - -# Configuration of the "jvm" context for null -jvm.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread -jvm.period=10 - -# Configuration of the "jvm" context for file -# jvm.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext -# jvm.fileName=/tmp/metrics_jvm.log - -# Configuration of the "jvm" context for ganglia -# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) -# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext -# jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 -# jvm.period=10 -# jvm.servers=GMETADHOST_IP:8649 - -# Configuration of the "rpc" context for null -rpc.class=org.apache.hadoop.metrics.spi.NullContextWithUpdateThread -rpc.period=10 - -# Configuration of the "rpc" context for file -# rpc.class=org.apache.hadoop.hbase.metrics.file.TimeStampingFileContext -# rpc.fileName=/tmp/metrics_rpc.log - -# Configuration of the "rpc" context for ganglia -# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) -# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext -# rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 -# rpc.period=10 -# rpc.servers=GMETADHOST_IP:8649 - -# Configuration of the "rest" context for ganglia -# Pick one: Ganglia 3.0 (former) or Ganglia 3.1 (latter) -# rest.class=org.apache.hadoop.metrics.ganglia.GangliaContext -# rest.class=org.apache.hadoop.metrics.ganglia.GangliaContext31 -# rest.period=10 -# rest.servers=GMETADHOST_IP:8649 diff --git a/titan-accumulo/src/test/config/hbase-env.sh b/titan-accumulo/src/test/config/hbase-env.sh deleted file mode 100644 index 13e58f8ef5..0000000000 --- a/titan-accumulo/src/test/config/hbase-env.sh +++ /dev/null @@ -1,94 +0,0 @@ -# -#/** -# * Copyright 2007 The Apache Software Foundation -# * -# * Licensed to the Apache Software Foundation (ASF) under one -# * or more contributor license agreements. See the NOTICE file -# * distributed with this work for additional information -# * regarding copyright ownership. The ASF licenses this file -# * to you under the Apache License, Version 2.0 (the -# * "License"); you may not use this file except in compliance -# * with the License. You may obtain a copy of the License at -# * -# * http://www.apache.org/licenses/LICENSE-2.0 -# * -# * Unless required by applicable law or agreed to in writing, software -# * distributed under the License is distributed on an "AS IS" BASIS, -# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# * See the License for the specific language governing permissions and -# * limitations under the License. -# */ - -# Set environment variables here. - -# The java implementation to use. Java 1.6 required. -# export JAVA_HOME=/usr/java/jdk1.6.0/ - -# Extra Java CLASSPATH elements. Optional. -export HBASE_CLASSPATH=$HBASE_CLASSPATH:./target/test-lib/* - -# The maximum amount of heap to use, in MB. Default is 1000. -# export HBASE_HEAPSIZE=1000 - -# Extra Java runtime options. -# Below are what we set by default. May only work with SUN JVM. -# For more on why as well as other possible settings, -# see http://wiki.apache.org/hadoop/PerformanceTuning -export HBASE_OPTS="$HBASE_OPTS -XX:+UseConcMarkSweepGC" - -# Uncomment below to enable java garbage collection logging in the .out file. -# export HBASE_OPTS="$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps $HBASE_GC_OPTS" - -# Uncomment below (along with above GC logging) to put GC information in its own logfile (will set HBASE_GC_OPTS) -# export HBASE_USE_GC_LOGFILE=true - - -# Uncomment below if you intend to use the EXPERIMENTAL off heap cache. -# export HBASE_OPTS="$HBASE_OPTS -XX:MaxDirectMemorySize=" -# Set hbase.offheapcache.percentage in hbase-site.xml to a nonzero value. - - -# Uncomment and adjust to enable JMX exporting -# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access. -# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html -# -# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false" -# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10101" -# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10102" -# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103" -# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104" - -# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default. -# export HBASE_REGIONSERVERS=${HBASE_HOME}/conf/regionservers - -# File naming hosts on which backup HMaster will run. $HBASE_HOME/conf/backup-masters by default. -# export HBASE_BACKUP_MASTERS=${HBASE_HOME}/conf/backup-masters - -# Extra ssh options. Empty by default. -# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR" - -# Where log files are stored. $HBASE_HOME/logs by default. -# export HBASE_LOG_DIR=${HBASE_HOME}/logs - -# Enable remote JDWP debugging of major HBase processes. Meant for Core Developers -# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8070" -# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8071" -# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8072" -# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=8073" - -# A string representing this instance of hbase. $USER by default. -# export HBASE_IDENT_STRING=$USER - -# The scheduling priority for daemon processes. See 'man nice'. -# export HBASE_NICENESS=10 - -# The directory where pid files are stored. /tmp by default. -# export HBASE_PID_DIR=/var/hadoop/pids - -# Seconds to sleep between slave commands. Unset by default. This -# can be useful in large clusters, where, e.g., slave rsyncs can -# otherwise arrive faster than the master can service them. -# export HBASE_SLAVE_SLEEP=0.1 - -# Tell HBase whether it should manage it's own instance of Zookeeper or not. -# export HBASE_MANAGES_ZK=true diff --git a/titan-accumulo/src/test/config/hbase-policy.xml b/titan-accumulo/src/test/config/hbase-policy.xml deleted file mode 100644 index e45f23c962..0000000000 --- a/titan-accumulo/src/test/config/hbase-policy.xml +++ /dev/null @@ -1,53 +0,0 @@ - - - - - - - security.client.protocol.acl - * - ACL for HRegionInterface protocol implementations (ie. - clients talking to HRegionServers) - The ACL is a comma-separated list of user and group names. The user and - group list is separated by a blank. For e.g. "alice,bob users,wheel". - A special value of "*" means all users are allowed. - - - - security.admin.protocol.acl - * - ACL for HMasterInterface protocol implementation (ie. - clients talking to HMaster for admin operations). - The ACL is a comma-separated list of user and group names. The user and - group list is separated by a blank. For e.g. "alice,bob users,wheel". - A special value of "*" means all users are allowed. - - - - security.masterregion.protocol.acl - * - ACL for HMasterRegionInterface protocol implementations - (for HRegionServers communicating with HMaster) - The ACL is a comma-separated list of user and group names. The user and - group list is separated by a blank. For e.g. "alice,bob users,wheel". - A special value of "*" means all users are allowed. - - diff --git a/titan-accumulo/src/test/config/hbase-site.xml b/titan-accumulo/src/test/config/hbase-site.xml deleted file mode 100644 index bf23d12c61..0000000000 --- a/titan-accumulo/src/test/config/hbase-site.xml +++ /dev/null @@ -1,38 +0,0 @@ - - - - - - - hbase.rootdir - ./src/test/titan-hbase-test-data - - - hbase.zookeeper.property.dataDir - ./src/test/titan-zookeeper-test - - - hbase.master - local - - diff --git a/titan-accumulo/src/test/config/log4j.properties b/titan-accumulo/src/test/config/log4j.properties deleted file mode 100644 index f6c3305178..0000000000 --- a/titan-accumulo/src/test/config/log4j.properties +++ /dev/null @@ -1,74 +0,0 @@ -# Define some default values that can be overridden by system properties -hbase.root.logger=INFO,console -hbase.security.logger=INFO,console -hbase.log.dir=. -hbase.log.file=hbase.log - -# Define the root logger to the system property "hbase.root.logger". -log4j.rootLogger=${hbase.root.logger} - -# Logging Threshold -log4j.threshold=ALL - -# -# Daily Rolling File Appender -# -log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender -log4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file} - -# Rollver at midnight -log4j.appender.DRFA.DatePattern=.yyyy-MM-dd - -# 30-day backup -#log4j.appender.DRFA.MaxBackupIndex=30 -log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout - -# Pattern format: Date LogLevel LoggerName LogMessage -log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n - -# Debugging Pattern format -#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n - -# -# Security audit appender -# -hbase.security.log.file=SecurityAuth.audit -log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender -log4j.appender.DRFAS.File=${hbase.log.dir}/${hbase.security.log.file} -log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout -log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n -log4j.category.SecurityLogger=${hbase.security.logger} -log4j.additivity.SecurityLogger=false - -# -# Null Appender -# -log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender - -# -# console -# Add "console" to rootlogger above if you want to use this -# -log4j.appender.console=org.apache.log4j.ConsoleAppender -log4j.appender.console.target=System.err -log4j.appender.console.layout=org.apache.log4j.PatternLayout -log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n - -# Custom Logging levels - -log4j.logger.org.apache.zookeeper=INFO -#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG -log4j.logger.org.apache.hadoop.hbase=DEBUG -# Make these two classes INFO-level. Make them DEBUG to see more zk debug. -log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=INFO -log4j.logger.org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher=INFO -#log4j.logger.org.apache.hadoop.dfs=DEBUG -# Set this class to log INFO only otherwise its OTT - -# Uncomment this line to enable tracing on _every_ RPC call (this can be a lot of output) -#log4j.logger.org.apache.hadoop.ipc.HBaseServer.trace=DEBUG - -# Uncomment the below if you want to remove logging of client region caching' -# and scan of .META. messages -# log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation=INFO -# log4j.logger.org.apache.hadoop.hbase.client.MetaScanner=INFO diff --git a/titan-accumulo/src/test/config/regionservers b/titan-accumulo/src/test/config/regionservers deleted file mode 100644 index 7b9ad531d2..0000000000 --- a/titan-accumulo/src/test/config/regionservers +++ /dev/null @@ -1 +0,0 @@ -127.0.0.1 diff --git a/titan-accumulo/src/test/config/whirr-hbase.properties b/titan-accumulo/src/test/config/whirr-hbase.properties deleted file mode 100644 index a9ced05fb7..0000000000 --- a/titan-accumulo/src/test/config/whirr-hbase.properties +++ /dev/null @@ -1,36 +0,0 @@ -# A Whirr Receipe for HBase + Hadoop -# Read the Configuration Guide for more info: -# http://whirr.apache.org/docs/latest/configuration-guide.html - -# Change the cluster name here -whirr.cluster-name=hbase-hadoop - -# Change the number of machines in the cluster here -whirr.instance-templates=1 zookeeper+hadoop-namenode+hadoop-jobtracker+hbase-master,3 hadoop-datanode+hadoop-tasktracker+hbase-regionserver - -# replication level should not be higher than number of data nodes -hbase-site.dfs.replication=2 - -# For EC2 set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables. -whirr.provider=aws-ec2 -whirr.identity=${env:AWS_ACCESS_KEY_ID} -whirr.credential=${env:AWS_SECRET_ACCESS_KEY} - -# The size of the instance to use. See http://aws.amazon.com/ec2/instance-types/ -whirr.hardware-id=m1.large -# Ubuntu 10.04 LTS Lucid. See http://alestic.com/ -whirr.image-id=us-east-1/ami-da0cf8b3 -# If you choose a different location, make sure whirr.image-id is updated too -whirr.location-id=us-east-1 - -# By default use the user system SSH keys. Override them here. -# whirr.private-key-file=${sys:user.home}/.ssh/id_rsa -# whirr.public-key-file=${whirr.private-key-file}.pub - -whirr.hbase.tarball.url=http://archive.apache.org/dist/hbase/hbase-0.94.1/hbase-0.94.1.tar.gz -whirr.hadoop.tarball.url=http://archive.apache.org/dist/hadoop/core/hadoop-1.0.3/hadoop-1.0.3.tar.gz - -# Options for the hbase master & regionserver processes -#hbase-env.HBASE_MASTER_OPTS=-Xms1000m -Xmx1000m -Xmn256m -XX:+UseConcMarkSweepGC -XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -Xloggc:/data/hbase/logs/hbase-master-gc.log -#hbase-env.HBASE_REGIONSERVER_OPTS=-Xms2000m -Xmx2000m -Xmn256m -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=88 -XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -Xloggc:/data/hbase/logs/hbase-regionserver-gc.log - From 5855507211cfed1cf391548f0f6909cbf67b0297 Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Sun, 16 Jun 2013 22:16:22 -0400 Subject: [PATCH 13/50] Add GraphOfTheGods example as Accumulo/Lucene graph. --- .../accumulo/GraphOfTheGodsFactory.java | 161 ++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 titan-accumulo/src/main/java/com/thinkaurelius/titan/example/accumulo/GraphOfTheGodsFactory.java diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/example/accumulo/GraphOfTheGodsFactory.java b/titan-accumulo/src/main/java/com/thinkaurelius/titan/example/accumulo/GraphOfTheGodsFactory.java new file mode 100644 index 0000000000..1e183a7acc --- /dev/null +++ b/titan-accumulo/src/main/java/com/thinkaurelius/titan/example/accumulo/GraphOfTheGodsFactory.java @@ -0,0 +1,161 @@ +package com.thinkaurelius.titan.example.accumulo; + +import com.thinkaurelius.titan.core.TitanFactory; +import com.thinkaurelius.titan.core.TitanGraph; +import com.thinkaurelius.titan.core.TitanKey; +import com.thinkaurelius.titan.core.attribute.Geoshape; +import com.thinkaurelius.titan.diskstorage.accumulo.AccumuloStoreManager; +import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration; +import com.tinkerpop.blueprints.Direction; +import com.tinkerpop.blueprints.Edge; +import com.tinkerpop.blueprints.Vertex; +import com.tinkerpop.blueprints.util.ElementHelper; +import org.apache.commons.configuration.BaseConfiguration; +import org.apache.commons.configuration.Configuration; +import java.io.File; +import java.io.IOException; + +/** + * Example Graph factory that creates a {@link TitanGraph} based on roman + * mythology. Used in the documentation examples and tutorials. + * + * @author Marko A. Rodriguez (http://markorodriguez.com) + */ +public class GraphOfTheGodsFactory { + + public static final String INDEX_NAME = "search"; + + public static void main(String[] args) throws IOException { + TitanGraph graph; + + System.out.println("Creating graph ... "); + graph = open(args[0]); + System.out.println("Loading graph ... "); + load(graph); + System.out.println("Done!"); + } + + public static TitanGraph create(final String directory) { + TitanGraph graph; + + graph = open(directory); + load(graph); + return graph; + } + + public static TitanGraph open(final String directory) { + BaseConfiguration config = new BaseConfiguration(); + Configuration storage = config.subset(GraphDatabaseConfiguration.STORAGE_NAMESPACE); + // configuring local backend + storage.setProperty(GraphDatabaseConfiguration.STORAGE_BACKEND_KEY, + "com.thinkaurelius.titan.diskstorage.accumulo.AccumuloStoreManager"); + storage.setProperty(GraphDatabaseConfiguration.HOSTNAME_KEY, "localhost"); + + Configuration accumulo = storage.subset(AccumuloStoreManager.ACCUMULO_CONFIGURATION_NAMESPACE); + + accumulo.addProperty(AccumuloStoreManager.ACCUMULO_INTSANCE_KEY, "EtCloud"); + accumulo.addProperty(GraphDatabaseConfiguration.HOSTNAME_KEY, "localhost"); + + accumulo.addProperty(AccumuloStoreManager.ACCUMULO_USER_KEY, "root"); + accumulo.addProperty(AccumuloStoreManager.ACCUMULO_PASSWORD_KEY, "bobross"); + // configuring elastic search index + Configuration index = storage.subset(GraphDatabaseConfiguration.INDEX_NAMESPACE).subset(INDEX_NAME); + index.setProperty(GraphDatabaseConfiguration.INDEX_BACKEND_KEY, "lucene"); + /* + index.setProperty("local-mode", true); + index.setProperty("client-only", false); + */ + index.setProperty(GraphDatabaseConfiguration.STORAGE_DIRECTORY_KEY, directory + File.separator + "lucene"); + + TitanGraph graph = TitanFactory.open(config); + + return graph; + } + + public static void load(final TitanGraph graph) { + + graph.makeType().name("name").dataType(String.class).indexed(Vertex.class).unique(Direction.BOTH).makePropertyKey(); + graph.makeType().name("age").dataType(Integer.class).indexed(INDEX_NAME, Vertex.class).unique(Direction.OUT).makePropertyKey(); + graph.makeType().name("type").dataType(String.class).unique(Direction.OUT).makePropertyKey(); + + final TitanKey time = graph.makeType().name("time").dataType(Integer.class).unique(Direction.OUT).makePropertyKey(); + final TitanKey reason = graph.makeType().name("reason").dataType(String.class).indexed(INDEX_NAME, Edge.class).unique(Direction.OUT).makePropertyKey(); + graph.makeType().name("place").dataType(Geoshape.class).indexed(INDEX_NAME, Edge.class).unique(Direction.OUT).makePropertyKey(); + + graph.makeType().name("father").unique(Direction.OUT).makeEdgeLabel(); + graph.makeType().name("mother").unique(Direction.OUT).makeEdgeLabel(); + graph.makeType().name("battled").primaryKey(time).makeEdgeLabel(); + graph.makeType().name("lives").signature(reason).makeEdgeLabel(); + graph.makeType().name("pet").makeEdgeLabel(); + graph.makeType().name("brother").makeEdgeLabel(); + + graph.commit(); + + // vertices + + Vertex saturn = graph.addVertex(null); + saturn.setProperty("name", "saturn"); + saturn.setProperty("age", 10000); + saturn.setProperty("type", "titan"); + + Vertex sky = graph.addVertex(null); + ElementHelper.setProperties(sky, "name", "sky", "type", "location"); + + Vertex sea = graph.addVertex(null); + ElementHelper.setProperties(sea, "name", "sea", "type", "location"); + + Vertex jupiter = graph.addVertex(null); + ElementHelper.setProperties(jupiter, "name", "jupiter", "age", 5000, "type", "god"); + + Vertex neptune = graph.addVertex(null); + ElementHelper.setProperties(neptune, "name", "neptune", "age", 4500, "type", "god"); + + Vertex hercules = graph.addVertex(null); + ElementHelper.setProperties(hercules, "name", "hercules", "age", 30, "type", "demigod"); + + Vertex alcmene = graph.addVertex(null); + ElementHelper.setProperties(alcmene, "name", "alcmene", "age", 45, "type", "human"); + + Vertex pluto = graph.addVertex(null); + ElementHelper.setProperties(pluto, "name", "pluto", "age", 4000, "type", "god"); + + Vertex nemean = graph.addVertex(null); + ElementHelper.setProperties(nemean, "name", "nemean", "type", "monster"); + + Vertex hydra = graph.addVertex(null); + ElementHelper.setProperties(hydra, "name", "hydra", "type", "monster"); + + Vertex cerberus = graph.addVertex(null); + ElementHelper.setProperties(cerberus, "name", "cerberus", "type", "monster"); + + Vertex tartarus = graph.addVertex(null); + ElementHelper.setProperties(tartarus, "name", "tartarus", "type", "location"); + + // edges + + jupiter.addEdge("father", saturn); + jupiter.addEdge("lives", sky).setProperty("reason", "loves fresh breezes"); + jupiter.addEdge("brother", neptune); + jupiter.addEdge("brother", pluto); + + neptune.addEdge("lives", sea).setProperty("reason", "loves waves"); + neptune.addEdge("brother", jupiter); + neptune.addEdge("brother", pluto); + + hercules.addEdge("father", jupiter); + hercules.addEdge("mother", alcmene); + ElementHelper.setProperties(hercules.addEdge("battled", nemean), "time", 1, "place", Geoshape.point(38.1f, 23.7f)); + ElementHelper.setProperties(hercules.addEdge("battled", hydra), "time", 2, "place", Geoshape.point(37.7f, 23.9f)); + ElementHelper.setProperties(hercules.addEdge("battled", cerberus), "time", 12, "place", Geoshape.point(39f, 22f)); + + pluto.addEdge("brother", jupiter); + pluto.addEdge("brother", neptune); + pluto.addEdge("lives", tartarus).setProperty("reason", "no fear of death"); + pluto.addEdge("pet", cerberus); + + cerberus.addEdge("lives", tartarus); + + // commit the transaction to disk + graph.commit(); + } +} From dc127194bc6103816e17ae26bb6be58c1c654516 Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Sun, 16 Jun 2013 23:14:15 -0400 Subject: [PATCH 14/50] Remove unused Bytes class. --- .../titan/diskstorage/accumulo/Bytes.java | 31 ------------------- 1 file changed, 31 deletions(-) delete mode 100644 titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/Bytes.java diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/Bytes.java b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/Bytes.java deleted file mode 100644 index 3433057efb..0000000000 --- a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/Bytes.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * To change this template, choose Tools | Templates - * and open the template in the editor. - */ -package com.thinkaurelius.titan.diskstorage.accumulo; - -/** - * - * @author edeprit - */ -public class Bytes { - - public static int compare(final byte[] left, final byte[] right) { - return compare(left, 0, left.length, right, 0, right.length); - } - - public static int compare(byte[] buffer1, int offset1, int length1, - byte[] buffer2, int offset2, int length2) { - // Bring WritableComparator code local - int end1 = offset1 + length1; - int end2 = offset2 + length2; - for (int i = offset1, j = offset2; i < end1 && j < end2; i++, j++) { - int a = (buffer1[i] & 0xff); - int b = (buffer2[j] & 0xff); - if (a != b) { - return a - b; - } - } - return length1 - length2; - } -} From 60282bc924a0f2c402e5de648cd65b00d061ce66 Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Thu, 20 Jun 2013 12:53:45 -0400 Subject: [PATCH 15/50] Configured unit test for mock Accumulo. --- titan-accumulo/pom.xml | 2 - .../accumulo/AccumuloStoreManager.java | 6 +- .../titan/AccumuloStorageSetup.java | 25 +- .../blueprints/AccumuloBlueprintsTest.java | 4 +- .../accumulo/AccumuloKeyColumnValueTest.java | 6 +- .../AccumuloLockKeyColumnValueStoreTest.java | 4 +- ...muloMultiWriteKeyColumnValueStoreTest.java | 5 +- .../accumulo/KeyColumnValueStoreTest.java | 502 ------------------ .../accumulo/MockAccumuloStoreManager.java | 26 + .../MultiWriteKeyColumnValueStoreTest.java | 414 --------------- 10 files changed, 58 insertions(+), 936 deletions(-) delete mode 100644 titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/KeyColumnValueStoreTest.java create mode 100644 titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/MockAccumuloStoreManager.java delete mode 100644 titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/MultiWriteKeyColumnValueStoreTest.java diff --git a/titan-accumulo/pom.xml b/titan-accumulo/pom.xml index 7447965d22..4e537523d5 100644 --- a/titan-accumulo/pom.xml +++ b/titan-accumulo/pom.xml @@ -35,7 +35,6 @@ org.apache.hadoop hadoop-core - 1.0.3 @@ -53,7 +52,6 @@ **/*Test.java - **/AccumuloGraphConcurrentTest.java **/Internal*.java **/*Suite.java diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java index 690592ac9c..192d111ba9 100644 --- a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java +++ b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java @@ -84,7 +84,7 @@ public AccumuloStoreManager(Configuration config) throws StorageException { username = accumuloConfig.getString(ACCUMULO_USER_KEY); password = accumuloConfig.getString(ACCUMULO_PASSWORD_KEY); - instance = new ZooKeeperInstance(instanceName, zooKeepers); + instance = createInstance(instanceName, zooKeepers); try { connector = instance.getConnector(username, password.getBytes()); } catch (AccumuloException ex) { @@ -107,6 +107,10 @@ public AccumuloStoreManager(Configuration config) throws StorageException { features.isDistributed = true; features.hasLocalKeyPartition = false; } + + protected Instance createInstance(String instanceName, String zooKeepers) { + return new ZooKeeperInstance(instanceName, zooKeepers); + } @Override public String toString() { diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java index eec46d2493..4b7e932e7d 100644 --- a/titan-accumulo/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java +++ b/titan-accumulo/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java @@ -1,5 +1,7 @@ package com.thinkaurelius.titan; +import com.thinkaurelius.titan.diskstorage.PermanentStorageException; +import com.thinkaurelius.titan.diskstorage.StorageException; import com.thinkaurelius.titan.diskstorage.accumulo.AccumuloStoreManager; import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration; import org.apache.commons.configuration.BaseConfiguration; @@ -8,6 +10,7 @@ import java.io.File; import java.io.IOException; +import java.lang.reflect.Constructor; public class AccumuloStorageSetup { @@ -52,6 +55,20 @@ public void run() { }); } + public static AccumuloStoreManager getAccumuloStoreManager() throws StorageException { + try { + Configuration config = getAccumuloStorageConfiguration(); + String backend = config.getString(GraphDatabaseConfiguration.STORAGE_BACKEND_KEY); + + Class cls = Class.forName(backend); + Constructor constructor = cls.getDeclaredConstructor(Configuration.class); + + return (AccumuloStoreManager) constructor.newInstance(config); + } catch (Exception ex) { + throw new PermanentStorageException(ex); + } + } + public static Configuration getAccumuloStorageConfiguration() { return getAccumuloGraphConfiguration() .subset(GraphDatabaseConfiguration.STORAGE_NAMESPACE); @@ -63,15 +80,15 @@ public static Configuration getAccumuloGraphConfiguration() { Configuration storageConfig = config.subset(GraphDatabaseConfiguration.STORAGE_NAMESPACE); storageConfig.addProperty(GraphDatabaseConfiguration.STORAGE_BACKEND_KEY, - "com.thinkaurelius.titan.diskstorage.accumulo.AccumuloStoreManager"); + "com.thinkaurelius.titan.diskstorage.accumulo.MockAccumuloStoreManager"); storageConfig.addProperty(GraphDatabaseConfiguration.HOSTNAME_KEY, "localhost"); Configuration accumuloConfig = storageConfig.subset(AccumuloStoreManager.ACCUMULO_CONFIGURATION_NAMESPACE); accumuloConfig.addProperty(AccumuloStoreManager.ACCUMULO_INTSANCE_KEY, "EtCloud"); - - accumuloConfig.addProperty(AccumuloStoreManager.ACCUMULO_USER_KEY, "root"); - accumuloConfig.addProperty(AccumuloStoreManager.ACCUMULO_PASSWORD_KEY, "bobross"); + + accumuloConfig.addProperty(AccumuloStoreManager.ACCUMULO_USER_KEY, "root"); + accumuloConfig.addProperty(AccumuloStoreManager.ACCUMULO_PASSWORD_KEY, ""); return config; } diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/blueprints/AccumuloBlueprintsTest.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/blueprints/AccumuloBlueprintsTest.java index 92040147f0..3c617bd858 100644 --- a/titan-accumulo/src/test/java/com/thinkaurelius/titan/blueprints/AccumuloBlueprintsTest.java +++ b/titan-accumulo/src/test/java/com/thinkaurelius/titan/blueprints/AccumuloBlueprintsTest.java @@ -35,9 +35,7 @@ public Graph generateGraph() { @Override public void cleanUp() throws StorageException { - AccumuloStoreManager s = new AccumuloStoreManager(AccumuloStorageSetup - .getAccumuloGraphConfiguration() - .subset(GraphDatabaseConfiguration.STORAGE_NAMESPACE)); + AccumuloStoreManager s = AccumuloStorageSetup.getAccumuloStoreManager(); s.clearStorage(); } diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueTest.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueTest.java index 5e36569327..c98b2cb64d 100644 --- a/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueTest.java +++ b/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueTest.java @@ -1,12 +1,11 @@ package com.thinkaurelius.titan.diskstorage.accumulo; import com.thinkaurelius.titan.AccumuloStorageSetup; +import com.thinkaurelius.titan.diskstorage.KeyColumnValueStoreTest; import com.thinkaurelius.titan.diskstorage.StorageException; import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyColumnValueStoreManager; import org.junit.BeforeClass; - import java.io.IOException; -import org.apache.commons.configuration.Configuration; public class AccumuloKeyColumnValueTest extends KeyColumnValueStoreTest { @@ -17,7 +16,6 @@ public static void startAccmulo() throws IOException { @Override public KeyColumnValueStoreManager openStorageManager() throws StorageException { - Configuration sc = AccumuloStorageSetup.getAccumuloStorageConfiguration(); - return new AccumuloStoreManager(sc); + return AccumuloStorageSetup.getAccumuloStoreManager(); } } diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloLockKeyColumnValueStoreTest.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloLockKeyColumnValueStoreTest.java index 55af9af7a6..1b24f551aa 100644 --- a/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloLockKeyColumnValueStoreTest.java +++ b/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloLockKeyColumnValueStoreTest.java @@ -4,7 +4,6 @@ import com.thinkaurelius.titan.diskstorage.LockKeyColumnValueStoreTest; import com.thinkaurelius.titan.diskstorage.StorageException; import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyColumnValueStoreManager; -import org.apache.commons.configuration.Configuration; import org.junit.BeforeClass; import java.io.IOException; @@ -17,7 +16,6 @@ public static void startAccumulo() throws IOException { @Override public KeyColumnValueStoreManager openStorageManager(int idx) throws StorageException { - Configuration sc = AccumuloStorageSetup.getAccumuloStorageConfiguration(); - return new AccumuloStoreManager(sc); + return AccumuloStorageSetup.getAccumuloStoreManager(); } } diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloMultiWriteKeyColumnValueStoreTest.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloMultiWriteKeyColumnValueStoreTest.java index e287d60aa6..eb3b92c3d9 100644 --- a/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloMultiWriteKeyColumnValueStoreTest.java +++ b/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloMultiWriteKeyColumnValueStoreTest.java @@ -1,12 +1,12 @@ package com.thinkaurelius.titan.diskstorage.accumulo; import com.thinkaurelius.titan.AccumuloStorageSetup; +import com.thinkaurelius.titan.diskstorage.MultiWriteKeyColumnValueStoreTest; import com.thinkaurelius.titan.diskstorage.StorageException; import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyColumnValueStoreManager; import org.junit.BeforeClass; import java.io.IOException; -import org.apache.commons.configuration.Configuration; public class AccumuloMultiWriteKeyColumnValueStoreTest extends MultiWriteKeyColumnValueStoreTest { @@ -17,7 +17,6 @@ public static void startAccumulo() throws IOException { @Override public KeyColumnValueStoreManager openStorageManager() throws StorageException { - Configuration sc = AccumuloStorageSetup.getAccumuloStorageConfiguration(); - return new AccumuloStoreManager(sc); + return AccumuloStorageSetup.getAccumuloStoreManager(); } } diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/KeyColumnValueStoreTest.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/KeyColumnValueStoreTest.java deleted file mode 100644 index 18234b104e..0000000000 --- a/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/KeyColumnValueStoreTest.java +++ /dev/null @@ -1,502 +0,0 @@ -package com.thinkaurelius.titan.diskstorage.accumulo; - - -import com.google.common.collect.Sets; -import com.thinkaurelius.titan.diskstorage.KeyColumn; -import com.thinkaurelius.titan.diskstorage.KeyColumnValueStoreUtil; -import com.thinkaurelius.titan.diskstorage.KeyValueStoreUtil; -import com.thinkaurelius.titan.diskstorage.StaticBuffer; -import com.thinkaurelius.titan.diskstorage.StorageException; -import com.thinkaurelius.titan.diskstorage.keycolumnvalue.*; -import com.thinkaurelius.titan.diskstorage.util.RecordIterator; -import com.thinkaurelius.titan.testutil.RandomGenerator; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.*; - -public abstract class KeyColumnValueStoreTest { - - private Logger log = LoggerFactory.getLogger(KeyColumnValueStoreTest.class); - - int numKeys = 50; - int numColumns = 5; - - protected String storeName = "testStore1"; - - public KeyColumnValueStoreManager manager; - public StoreTransaction tx; - public KeyColumnValueStore store; - - @Before - public void setUp() throws Exception { - openStorageManager().clearStorage(); - open(); - } - - public abstract KeyColumnValueStoreManager openStorageManager() throws StorageException; - - public void open() throws StorageException { - manager = openStorageManager(); - tx = manager.beginTransaction(ConsistencyLevel.DEFAULT); - store = manager.openDatabase(storeName); - } - - public void clopen() throws StorageException { - close(); - open(); - } - - @After - public void tearDown() throws Exception { - close(); - } - - public void close() throws StorageException { - if (tx != null) tx.commit(); - store.close(); - manager.close(); - } - - @Test - public void createDatabase() { - //Just setup and shutdown - } - - - public String[][] generateValues() { - return KeyValueStoreUtil.generateData(numKeys, numColumns); - } - - public void loadValues(String[][] values) throws StorageException { - for (int i = 0; i < values.length; i++) { - List entries = new ArrayList(); - for (int j = 0; j < values[i].length; j++) { - entries.add(new StaticBufferEntry(KeyValueStoreUtil.getBuffer(j), KeyValueStoreUtil.getBuffer(values[i][j]))); - } - store.mutate(KeyValueStoreUtil.getBuffer(i), entries, KeyColumnValueStore.NO_DELETIONS, tx); - } - } - - public Set deleteValues(int every) throws StorageException { - Set removed = new HashSet(); - int counter = 0; - for (int i = 0; i < numKeys; i++) { - List deletions = new ArrayList(); - for (int j = 0; j < numColumns; j++) { - counter++; - if (counter % every == 0) { - //remove - removed.add(new KeyColumn(i, j)); - deletions.add(KeyValueStoreUtil.getBuffer(j)); - } - } - store.mutate(KeyValueStoreUtil.getBuffer(i), KeyColumnValueStore.NO_ADDITIONS, deletions, tx); - } - return removed; - } - - public Set deleteKeys(int every) throws StorageException { - Set removed = new HashSet(); - for (int i = 0; i < numKeys; i++) { - if (i % every == 0) { - removed.add(i); - List deletions = new ArrayList(); - for (int j = 0; j < numColumns; j++) { - deletions.add(KeyValueStoreUtil.getBuffer(j)); - } - store.mutate(KeyValueStoreUtil.getBuffer(i), KeyColumnValueStore.NO_ADDITIONS, deletions, tx); - } - } - return removed; - } - - public void checkKeys(Set removed) throws StorageException { - for (int i = 0; i < numKeys; i++) { - if (removed.contains(i)) { - Assert.assertFalse(store.containsKey(KeyValueStoreUtil.getBuffer(i), tx)); - } else { - Assert.assertTrue(store.containsKey(KeyValueStoreUtil.getBuffer(i), tx)); - } - } - } - - public void checkValueExistence(String[][] values) throws StorageException { - checkValueExistence(values, new HashSet()); - } - - public void checkValueExistence(String[][] values, Set removed) throws StorageException { - for (int i = 0; i < numKeys; i++) { - for (int j = 0; j < numColumns; j++) { - boolean result = KCVSUtil.containsKeyColumn(store, KeyValueStoreUtil.getBuffer(i), KeyValueStoreUtil.getBuffer(j), tx); - if (removed.contains(new KeyColumn(i, j))) { - Assert.assertFalse(result); - } else { - Assert.assertTrue(result); - } - } - } - } - - public void checkValues(String[][] values) throws StorageException { - checkValues(values, new HashSet()); - } - - public void checkValues(String[][] values, Set removed) throws StorageException { - for (int i = 0; i < numKeys; i++) { - for (int j = 0; j < numColumns; j++) { - StaticBuffer result = KCVSUtil.get(store,KeyValueStoreUtil.getBuffer(i), KeyValueStoreUtil.getBuffer(j), tx); - if (removed.contains(new KeyColumn(i, j))) { - Assert.assertNull(result); - } else { - Assert.assertEquals(values[i][j], KeyValueStoreUtil.getString(result)); - } - } - } - - } - - @Test - public void storeAndRetrieve() throws StorageException { - String[][] values = generateValues(); - log.debug("Loading values..."); - loadValues(values); - //print(values); - log.debug("Checking values..."); - checkValueExistence(values); - checkValues(values); - } - - @Test - public void storeAndRetrievePerformance() throws StorageException { - int multiplier = 4; - int keys = 50*multiplier, columns = 2000; - String[][] values = KeyValueStoreUtil.generateData(keys,columns); - log.debug("Loading values: "+keys+"x"+columns); - long time = System.currentTimeMillis(); - loadValues(values); - System.out.println("Loading time (ms): " + (System.currentTimeMillis()-time)); - //print(values); - Random r = new Random(); - int trials = 500*multiplier; - int delta = 10; - log.debug("Reading values: "+trials+" trials"); - time = System.currentTimeMillis(); - for (int t=0;t deleted = deleteValues(7); - log.debug("Checking values..."); - checkValueExistence(values, deleted); - checkValues(values, deleted); - } - - @Test - public void deleteColumnsTest2() throws StorageException { - String[][] values = generateValues(); - log.debug("Loading values..."); - loadValues(values); - Set deleted = deleteValues(7); - clopen(); - log.debug("Checking values..."); - checkValueExistence(values, deleted); - checkValues(values, deleted); - } - - @Test - public void deleteKeys() throws StorageException { - String[][] values = generateValues(); - log.debug("Loading values..."); - loadValues(values); - Set deleted = deleteKeys(11); - clopen(); - checkKeys(deleted); - } - - @Test - public void scanTest() throws StorageException { - if (manager.getFeatures().supportsScan()) { - String[][] values = generateValues(); - loadValues(values); - RecordIterator iterator0 = store.getKeys(tx); - Assert.assertEquals(numKeys, KeyValueStoreUtil.count(iterator0)); - clopen(); - RecordIterator iterator1 = store.getKeys(tx); - RecordIterator iterator2 = store.getKeys(tx); - // The idea is to open an iterator without using it - // to make sure that closing a transaction will clean it up. - // (important for BerkeleyJE where leaving cursors open causes exceptions) - @SuppressWarnings("unused") - RecordIterator iterator3 = store.getKeys(tx); - Assert.assertEquals(numKeys, KeyValueStoreUtil.count(iterator1)); - Assert.assertEquals(numKeys, KeyValueStoreUtil.count(iterator2)); - } - } - - public void checkSlice(String[][] values, Set removed, int key, int start, int end, int limit) throws StorageException { - List entries; - if (limit <= 0) - entries = store.getSlice(new KeySliceQuery(KeyValueStoreUtil.getBuffer(key), KeyValueStoreUtil.getBuffer(start), KeyValueStoreUtil.getBuffer(end)), tx); - else - entries = store.getSlice(new KeySliceQuery(KeyValueStoreUtil.getBuffer(key), KeyValueStoreUtil.getBuffer(start), KeyValueStoreUtil.getBuffer(end), limit), tx); - - int pos = 0; - for (int i = start; i < end; i++) { - if (removed.contains(new KeyColumn(key, i))) continue; - if (limit <= 0 || pos < limit) { - Assert.assertTrue(entries.size()>pos); - Entry entry = entries.get(pos); - int col = KeyValueStoreUtil.getID(entry.getColumn()); - String str = KeyValueStoreUtil.getString(entry.getValue()); - Assert.assertEquals(i, col); - Assert.assertEquals(values[key][i], str); - } - pos++; - } - Assert.assertNotNull(entries); - if (limit > 0 && pos > limit) Assert.assertEquals(limit, entries.size()); - else Assert.assertEquals(pos, entries.size()); - } - - @Test - public void intervalTest1() throws StorageException { - String[][] values = generateValues(); - log.debug("Loading values..."); - loadValues(values); - Set deleted = Sets.newHashSet(); - clopen(); - int trails = 5000; - for (int t = 0; t < trails; t++) { - int key = RandomGenerator.randomInt(0, numKeys); - int start = RandomGenerator.randomInt(0, numColumns); - int end = RandomGenerator.randomInt(start, numColumns); - int limit = RandomGenerator.randomInt(1, 30); - checkSlice(values, deleted, key, start, end, limit); - checkSlice(values, deleted, key, start, end, -1); - } - } - - @Test - public void intervalTest2() throws StorageException { - String[][] values = generateValues(); - log.debug("Loading values..."); - loadValues(values); - Set deleted = deleteValues(7); - clopen(); - int trails = 5000; - for (int t = 0; t < trails; t++) { - int key = RandomGenerator.randomInt(0, numKeys); - int start = RandomGenerator.randomInt(0, numColumns); - int end = RandomGenerator.randomInt(start, numColumns); - int limit = RandomGenerator.randomInt(1, 30); - checkSlice(values, deleted, key, start, end, limit); - checkSlice(values, deleted, key, start, end, -1); - } - } - - - @Test - public void getNonExistentKeyReturnsNull() throws Exception { - StoreTransaction txn = manager.beginTransaction(ConsistencyLevel.DEFAULT); - Assert.assertEquals(null, KeyColumnValueStoreUtil.get(store, txn, 0, "col0")); - Assert.assertEquals(null, KeyColumnValueStoreUtil.get(store, txn, 0, "col1")); - txn.commit(); - } - - @Test - public void insertingGettingAndDeletingSimpleDataWorks() throws Exception { - StoreTransaction txn = manager.beginTransaction(ConsistencyLevel.DEFAULT); - KeyColumnValueStoreUtil.insert(store, txn, 0, "col0", "val0"); - KeyColumnValueStoreUtil.insert(store, txn, 0, "col1", "val1"); - txn.commit(); - - txn = manager.beginTransaction(ConsistencyLevel.DEFAULT); - Assert.assertEquals("val0", KeyColumnValueStoreUtil.get(store, txn, 0, "col0")); - Assert.assertEquals("val1", KeyColumnValueStoreUtil.get(store, txn, 0, "col1")); - KeyColumnValueStoreUtil.delete(store, txn, 0, "col0"); - KeyColumnValueStoreUtil.delete(store, txn, 0, "col1"); - txn.commit(); - - txn = manager.beginTransaction(ConsistencyLevel.DEFAULT); - Assert.assertEquals(null, KeyColumnValueStoreUtil.get(store, txn, 0, "col0")); - Assert.assertEquals(null, KeyColumnValueStoreUtil.get(store, txn, 0, "col1")); - txn.commit(); - } - -// @Test -// public void getSliceNoLimit() throws Exception { -// CassandraThriftStoreManager manager = new CassandraThriftStoreManager(keyspace); -// CassandraThriftKeyColumnValueStore store = -// manager.openDatabase(dbName); -// -// StoreTransaction txn = manager.beginTransaction(); -// KeyColumnValueStoreUtil.insert(store, txn, "key0", "col0", "val0"); -// KeyColumnValueStoreUtil.insert(store, txn, "key0", "col1", "val1"); -// txn.commit(); -// -// txn = manager.beginTransaction(); -// ByteBuffer key0 = KeyColumnValueStoreUtil.stringToByteBuffer("key0"); -// ByteBuffer col0 = KeyColumnValueStoreUtil.stringToByteBuffer("col0"); -// ByteBuffer col2 = KeyColumnValueStoreUtil.stringToByteBuffer("col2"); -// List entries = store.getSlice(key0, col0, col2, txn); -// assertNotNull(entries); -// assertEquals(2, entries.size()); -// assertEquals("col0", KeyColumnValueStoreUtil.byteBufferToString(entries.get(0).getColumn())); -// assertEquals("val0", KeyColumnValueStoreUtil.byteBufferToString(entries.get(0).getValue())); -// assertEquals("col1", KeyColumnValueStoreUtil.byteBufferToString(entries.get(1).getColumn())); -// assertEquals("val1", KeyColumnValueStoreUtil.byteBufferToString(entries.get(1).getValue())); -// -// txn.commit(); -// -// store.shutdown(); -// manager.shutdown(); -// } - - @Test - public void getSliceRespectsColumnLimit() throws Exception { - StoreTransaction txn = manager.beginTransaction(ConsistencyLevel.DEFAULT); - StaticBuffer key = KeyColumnValueStoreUtil.longToByteBuffer(0); - - final int cols = 1024; - - List entries = new LinkedList(); - for (int i = 0; i < cols; i++) { - StaticBuffer col = KeyColumnValueStoreUtil.longToByteBuffer(i); - entries.add(new StaticBufferEntry(col, col)); - } - store.mutate(key, entries, KeyColumnValueStore.NO_DELETIONS, txn); - txn.commit(); - - txn = manager.beginTransaction(ConsistencyLevel.DEFAULT); - StaticBuffer columnStart = KeyColumnValueStoreUtil.longToByteBuffer(0); - StaticBuffer columnEnd = KeyColumnValueStoreUtil.longToByteBuffer(cols); /* - * When limit is greater than or equal to the matching column count, - * all matching columns must be returned. - */ - List result = - store.getSlice(new KeySliceQuery(key, columnStart, columnEnd, cols), txn); - Assert.assertEquals(cols, result.size()); - Assert.assertEquals(entries, result); - result = - store.getSlice(new KeySliceQuery(key, columnStart, columnEnd, cols + 10), txn); - Assert.assertEquals(cols, result.size()); - Assert.assertEquals(entries, result); - - /* - * When limit is less the matching column count, the columns up to the - * limit (ordered bytewise) must be returned. - */ - result = - store.getSlice(new KeySliceQuery(key, columnStart, columnEnd, cols - 1), txn); - Assert.assertEquals(cols - 1, result.size()); - entries.remove(entries.size() - 1); - Assert.assertEquals(entries, result); - result = - store.getSlice(new KeySliceQuery(key, columnStart, columnEnd, 1), txn); - Assert.assertEquals(1, result.size()); - List firstEntrySingleton = Arrays.asList(entries.get(0)); - Assert.assertEquals(firstEntrySingleton, result); - txn.commit(); - } - - @Test - public void getSliceRespectsAllBoundsInclusionArguments() throws Exception { - // Test case where endColumn=startColumn+1 - StaticBuffer key = KeyColumnValueStoreUtil.longToByteBuffer(0); - StaticBuffer columnBeforeStart = KeyColumnValueStoreUtil.longToByteBuffer(776); - StaticBuffer columnStart = KeyColumnValueStoreUtil.longToByteBuffer(777); - StaticBuffer columnEnd = KeyColumnValueStoreUtil.longToByteBuffer(778); - StaticBuffer columnAfterEnd = KeyColumnValueStoreUtil.longToByteBuffer(779); - - // First insert four test Entries - StoreTransaction txn = manager.beginTransaction(ConsistencyLevel.DEFAULT); - List entries = Arrays.asList( - (Entry)new StaticBufferEntry(columnBeforeStart, columnBeforeStart), - new StaticBufferEntry(columnStart, columnStart), - new StaticBufferEntry(columnEnd, columnEnd), - new StaticBufferEntry(columnAfterEnd, columnAfterEnd)); - store.mutate(key, entries, KeyColumnValueStore.NO_DELETIONS, txn); - txn.commit(); - - // getSlice() with only start inclusive - txn = manager.beginTransaction(ConsistencyLevel.DEFAULT); - List result = store.getSlice(new KeySliceQuery(key, columnStart, columnEnd), txn); - Assert.assertEquals(1, result.size()); - Assert.assertEquals(777, KeyColumnValueStoreUtil.bufferToLong(result.get(0).getColumn())); - txn.commit(); - - } - - - @Test - public void containsKeyReturnsTrueOnExtantKey() throws Exception { - StaticBuffer key1 = KeyColumnValueStoreUtil.longToByteBuffer(1); - StoreTransaction txn = manager.beginTransaction(ConsistencyLevel.DEFAULT); - Assert.assertFalse(store.containsKey(key1, txn)); - KeyColumnValueStoreUtil.insert(store, txn, 1, "c", "v"); - txn.commit(); - - txn = manager.beginTransaction(ConsistencyLevel.DEFAULT); - Assert.assertTrue(store.containsKey(key1, txn)); - txn.commit(); - } - - - @Test - public void containsKeyReturnsFalseOnNonexistentKey() throws Exception { - StoreTransaction txn = manager.beginTransaction(ConsistencyLevel.DEFAULT); - StaticBuffer key1 = KeyColumnValueStoreUtil.longToByteBuffer(1); - Assert.assertFalse(store.containsKey(key1, txn)); - txn.commit(); - } - - - @Test - public void containsKeyColumnReturnsFalseOnNonexistentInput() throws Exception { - StoreTransaction txn = manager.beginTransaction(ConsistencyLevel.DEFAULT); - StaticBuffer key1 = KeyColumnValueStoreUtil.longToByteBuffer(1); - StaticBuffer c = KeyColumnValueStoreUtil.stringToByteBuffer("c"); - Assert.assertFalse(KCVSUtil.containsKeyColumn(store,key1, c, txn)); - txn.commit(); - } - - @Test - public void containsKeyColumnReturnsTrueOnExtantInput() throws Exception { - StoreTransaction txn = manager.beginTransaction(ConsistencyLevel.DEFAULT); - KeyColumnValueStoreUtil.insert(store, txn, 1, "c", "v"); - txn.commit(); - - txn = manager.beginTransaction(ConsistencyLevel.DEFAULT); - StaticBuffer key1 = KeyColumnValueStoreUtil.longToByteBuffer(1); - StaticBuffer c = KeyColumnValueStoreUtil.stringToByteBuffer("c"); - Assert.assertTrue(KCVSUtil.containsKeyColumn(store,key1, c, txn)); - txn.commit(); - } -} - diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/MockAccumuloStoreManager.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/MockAccumuloStoreManager.java new file mode 100644 index 0000000000..0174f25226 --- /dev/null +++ b/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/MockAccumuloStoreManager.java @@ -0,0 +1,26 @@ +/* + * To change this template, choose Tools | Templates + * and open the template in the editor. + */ +package com.thinkaurelius.titan.diskstorage.accumulo; + +import com.thinkaurelius.titan.diskstorage.StorageException; +import org.apache.accumulo.core.client.Instance; +import org.apache.accumulo.core.client.mock.MockInstance; +import org.apache.commons.configuration.Configuration; + +/** + * + * @author edeprit + */ +public class MockAccumuloStoreManager extends AccumuloStoreManager { + + public MockAccumuloStoreManager(Configuration config) throws StorageException { + super(config); + } + + @Override + protected Instance createInstance(String instanceName, String zooKeepers) { + return new MockInstance(instanceName); + } +} diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/MultiWriteKeyColumnValueStoreTest.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/MultiWriteKeyColumnValueStoreTest.java deleted file mode 100644 index 0b960f8d7e..0000000000 --- a/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/MultiWriteKeyColumnValueStoreTest.java +++ /dev/null @@ -1,414 +0,0 @@ -package com.thinkaurelius.titan.diskstorage.accumulo; - -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.thinkaurelius.titan.diskstorage.KeyColumnValueStoreUtil; -import com.thinkaurelius.titan.diskstorage.StaticBuffer; -import com.thinkaurelius.titan.diskstorage.StorageException; -import com.thinkaurelius.titan.diskstorage.keycolumnvalue.*; -import com.thinkaurelius.titan.diskstorage.util.StaticArrayBuffer; - -import static com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyColumnValueStore.*; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.*; - -public abstract class MultiWriteKeyColumnValueStoreTest { - - private Logger log = LoggerFactory.getLogger(MultiWriteKeyColumnValueStoreTest.class); - - int numKeys = 50; - int numColumns = 5; - - int bufferSize = 20; - - protected String storeName1 = "testStore1"; - private KeyColumnValueStore store1; - protected String storeName2 = "testStore2"; - private KeyColumnValueStore store2; - - - public KeyColumnValueStoreManager manager; - public StoreTransaction tx; - - - private Random rand = new Random(10); - - @Before - public void setUp() throws Exception { - openStorageManager().clearStorage(); - open(); - } - - public abstract KeyColumnValueStoreManager openStorageManager() throws StorageException; - - public void open() throws StorageException { - manager = openStorageManager(); - Assert.assertTrue(manager.getFeatures().supportsBatchMutation()); - tx = new BufferTransaction(manager.beginTransaction(ConsistencyLevel.DEFAULT), manager, bufferSize, 1, 0); - store1 = new BufferedKeyColumnValueStore(manager.openDatabase(storeName1), true); - store2 = new BufferedKeyColumnValueStore(manager.openDatabase(storeName2), true); - - } - - public void clopen() throws StorageException { - close(); - open(); - } - - @After - public void tearDown() throws Exception { - close(); - } - - public void close() throws StorageException { - if (tx != null) tx.commit(); - if (null != store1) store1.close(); - if (null != store2) store2.close(); - if (null != manager) manager.close(); - } - - @Test - public void deletionsAppliedBeforeAdditions() throws StorageException { - - StaticBuffer b1 = KeyColumnValueStoreUtil.longToByteBuffer(1); - - Assert.assertNull(KCVSUtil.get(store1,b1, b1, tx)); - - List additions = Arrays.asList(new StaticBufferEntry(b1, b1)); - - List deletions = Arrays.asList(b1); - - Map combination = new HashMap(1); - Map deleteOnly = new HashMap(1); - Map addOnly = new HashMap(1); - - combination.put(b1, new KCVMutation(additions, deletions)); - deleteOnly.put(b1, new KCVMutation(KeyColumnValueStore.NO_ADDITIONS, deletions)); - addOnly.put(b1, new KCVMutation(additions, KeyColumnValueStore.NO_DELETIONS)); - - store1.mutate(b1, additions, deletions, tx); - tx.flush(); - - StaticBuffer result = KCVSUtil.get(store1,b1, b1, tx); - - Assert.assertEquals(b1, result); - - store1.mutate(b1, NO_ADDITIONS, deletions, tx); - tx.flush(); - - for (int i = 0; i < 100; i++) { - StaticBuffer n = KCVSUtil.get(store1,b1, b1, tx); - Assert.assertNull(n); - store1.mutate(b1, additions, NO_DELETIONS, tx); - tx.flush(); - store1.mutate(b1, NO_ADDITIONS, deletions, tx); - tx.flush(); - n = KCVSUtil.get(store1,b1, b1, tx); - Assert.assertNull(n); - } - - for (int i = 0; i < 100; i++) { - store1.mutate(b1, NO_ADDITIONS, deletions, tx); - tx.flush(); - store1.mutate(b1, additions, NO_DELETIONS, tx); - tx.flush(); - Assert.assertEquals(b1, KCVSUtil.get(store1,b1, b1, tx)); - } - - for (int i = 0; i < 100; i++) { - store1.mutate(b1, additions, deletions, tx); - tx.flush(); - Assert.assertEquals(b1, KCVSUtil.get(store1,b1, b1, tx)); - } - } - - @Test - public void mutateManyWritesSameKeyOnMultipleCFs() throws StorageException { - - final long arbitraryLong = 42; - assert 0 < arbitraryLong; - - final StaticBuffer key = KeyColumnValueStoreUtil.longToByteBuffer(arbitraryLong * arbitraryLong); - final StaticBuffer val = KeyColumnValueStoreUtil.longToByteBuffer(arbitraryLong * arbitraryLong * arbitraryLong); - final StaticBuffer col = KeyColumnValueStoreUtil.longToByteBuffer(arbitraryLong); - final StaticBuffer nextCol = KeyColumnValueStoreUtil.longToByteBuffer(arbitraryLong + 1); - - final StoreTransaction directTx = manager.beginTransaction(ConsistencyLevel.DEFAULT); - - KCVMutation km = new KCVMutation( - ImmutableList.of(new StaticBufferEntry(col, val)), - ImmutableList.of()); - - Map keyColumnAndValue = ImmutableMap.of(key, km); - - Map> mutations = - ImmutableMap.of( - storeName1, keyColumnAndValue, - storeName2, keyColumnAndValue); - - manager.mutateMany(mutations, directTx); - - directTx.commit(); - - KeySliceQuery query = new KeySliceQuery(key, col, nextCol); - List expected = - ImmutableList.of(new StaticBufferEntry(col, val)); - - Assert.assertEquals(expected, store1.getSlice(query, tx)); - Assert.assertEquals(expected, store2.getSlice(query, tx)); - - } - - @Test - public void mutateManyStressTest() throws StorageException { - - Map> state = - new HashMap>(); - - int dels = 1024; - int adds = 4096; - - for (int round = 0; round < 5; round++) { - Map changes = mutateState(state, dels, adds); - - applyChanges(changes, store1, tx); - applyChanges(changes, store2, tx); - tx.flush(); - - int deletesExpected = 0 == round ? 0 : dels; - - int stateSizeExpected = adds + (adds - dels) * round; - - Assert.assertEquals(stateSizeExpected, checkThatStateExistsInStore(state, store1, round)); - Assert.assertEquals(deletesExpected, checkThatDeletionsApplied(changes, store1, round)); - - Assert.assertEquals(stateSizeExpected, checkThatStateExistsInStore(state, store2, round)); - Assert.assertEquals(deletesExpected, checkThatDeletionsApplied(changes, store2, round)); - } - } - - public void applyChanges(Map changes, KeyColumnValueStore store, StoreTransaction tx) throws StorageException { - for (Map.Entry change : changes.entrySet()) { - store.mutate(change.getKey(), change.getValue().getAdditions(), change.getValue().getDeletions(), tx); - } - } - - public int checkThatStateExistsInStore(Map> state, KeyColumnValueStore store, int round) throws StorageException { - int checked = 0; - - for (StaticBuffer key : state.keySet()) { - for (StaticBuffer col : state.get(key).keySet()) { - StaticBuffer val = state.get(key).get(col); - - Assert.assertEquals(val, KCVSUtil.get(store,key, col, tx)); - - checked++; - } - } - - log.debug("Checked existence of {} key-column-value triples on round {}", checked, round); - - return checked; - } - - public int checkThatDeletionsApplied(Map changes, KeyColumnValueStore store, int round) throws StorageException { - int checked = 0; - int skipped = 0; - - for (StaticBuffer key : changes.keySet()) { - KCVMutation m = changes.get(key); - - if (!m.hasDeletions()) - continue; - - List deletions = m.getDeletions(); - - List additions = m.getAdditions(); - - for (StaticBuffer col : deletions) { - - if (null != additions && additions.contains(new StaticBufferEntry(col, col))) { - skipped++; - continue; - } - - Assert.assertNull(KCVSUtil.get(store,key, col, tx)); - - checked++; - } - } - - log.debug("Checked absence of {} key-column-value deletions on round {} (skipped {})", new Object[]{checked, round, skipped}); - - return checked; - } - - /** - * Pseudorandomly change the supplied {@code state}. - *

- * This method removes {@code min(maxDeletionCount, S)} entries from the - * maps in {@code state.values()}, where {@code S} is the sum of the sizes - * of the maps in {@code state.values()}; this method then adds - * {@code additionCount} pseudorandomly generated entries spread across - * {@code state.values()}, potentially adding new keys to {@code state} - * since they are randomly generated. This method then returns a map of keys - * to Mutations representing the changes it has made to {@code state}. - * - * @param state Maps keys -> columns -> values - * @param maxDeletionCount Remove at most this many entries from state - * @param additionCount Add exactly this many entries to state - * @return A KCVMutation map - */ - public Map mutateState( - Map> state, - int maxDeletionCount, int additionCount) { - - final int keyLength = 8; - final int colLength = 16; - - Map result = new HashMap(); - - // deletion pass - int dels = 0; - - StaticBuffer key = null, col = null; - - Iterator keyIter = state.keySet().iterator(); - - while (keyIter.hasNext() && dels < maxDeletionCount) { - key = keyIter.next(); - - Iterator colIter = - state.get(key).keySet().iterator(); - - while (colIter.hasNext() && dels < maxDeletionCount) { - col = colIter.next(); - - if (!result.containsKey(key)) { - KCVMutation m = new KCVMutation(new LinkedList(), - new LinkedList()); - result.put(key, m); - } - - result.get(key).deletion(col); - - dels++; - - colIter.remove(); - - if (state.get(key).isEmpty()) { - assert !colIter.hasNext(); - keyIter.remove(); - } - } - } - - // addition pass - for (int i = 0; i < additionCount; i++) { - - while (true) { - byte keyBuf[] = new byte[keyLength]; - rand.nextBytes(keyBuf); - key = new StaticArrayBuffer(keyBuf); - - byte colBuf[] = new byte[colLength]; - rand.nextBytes(colBuf); - col = new StaticArrayBuffer(colBuf); - - if (!state.containsKey(key) || !state.get(key).containsKey(col)) { - break; - } - } - - if (!state.containsKey(key)) { - Map m = new HashMap(); - state.put(key, m); - } - - state.get(key).put(col, col); - - if (!result.containsKey(key)) { - KCVMutation m = new KCVMutation(new LinkedList(), - new LinkedList()); - result.put(key, m); - } - - result.get(key).addition(new StaticBufferEntry(col, col)); - - } - - return result; - } - - public Map generateMutation(int keyCount, int columnCount, Map deleteFrom) { - Map result = new HashMap(keyCount); - - Random keyRand = new Random(keyCount); - Random colRand = new Random(columnCount); - - final int keyLength = 8; - final int colLength = 6; - - Iterator> deleteIter = null; - List lastDeleteIterResult = null; - - if (null != deleteFrom) { - deleteIter = deleteFrom.entrySet().iterator(); - } - - for (int ik = 0; ik < keyCount; ik++) { - byte keyBuf[] = new byte[keyLength]; - keyRand.nextBytes(keyBuf); - StaticBuffer key = new StaticArrayBuffer(keyBuf); - - List additions = new LinkedList(); - List deletions = new LinkedList(); - - for (int ic = 0; ic < columnCount; ic++) { - - boolean deleteSucceeded = false; - if (null != deleteIter && 1 == ic % 2) { - - if (null == lastDeleteIterResult || lastDeleteIterResult.isEmpty()) { - while (deleteIter.hasNext()) { - Map.Entry ent = deleteIter.next(); - if (ent.getValue().hasAdditions() && !ent.getValue().getAdditions().isEmpty()) { - lastDeleteIterResult = ent.getValue().getAdditions(); - break; - } - } - } - - - if (null != lastDeleteIterResult && !lastDeleteIterResult.isEmpty()) { - Entry e = lastDeleteIterResult.get(0); - lastDeleteIterResult.remove(0); - deletions.add(e.getColumn()); - deleteSucceeded = true; - } - } - - if (!deleteSucceeded) { - byte colBuf[] = new byte[colLength]; - colRand.nextBytes(colBuf); - StaticBuffer col = new StaticArrayBuffer(colBuf); - - additions.add(new StaticBufferEntry(col, col)); - } - - } - - KCVMutation m = new KCVMutation(additions, deletions); - - result.put(key, m); - } - - return result; - } -} From 1788d5bfbf9aa63a4282303d3b4af293a9cac1ec Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Fri, 21 Jun 2013 11:01:58 -0400 Subject: [PATCH 16/50] Storage configuration persisted in Accumulo table. --- .../accumulo/AccumuloStoreManager.java | 55 ++--------- .../util/AccumuloStorageConfiguration.java | 98 +++++++++++++++++++ .../accumulo/{ => util}/MutablePair.java | 2 +- 3 files changed, 109 insertions(+), 46 deletions(-) create mode 100644 titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/util/AccumuloStorageConfiguration.java rename titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/{ => util}/MutablePair.java (97%) diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java index 192d111ba9..fb32ec7096 100644 --- a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java +++ b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java @@ -1,8 +1,10 @@ package com.thinkaurelius.titan.diskstorage.accumulo; +import com.thinkaurelius.titan.diskstorage.accumulo.util.MutablePair; import com.thinkaurelius.titan.diskstorage.PermanentStorageException; import com.thinkaurelius.titan.diskstorage.StaticBuffer; import com.thinkaurelius.titan.diskstorage.StorageException; +import com.thinkaurelius.titan.diskstorage.accumulo.util.AccumuloStorageConfiguration; import com.thinkaurelius.titan.diskstorage.common.DistributedStoreManager; import com.thinkaurelius.titan.diskstorage.keycolumnvalue.ConsistencyLevel; import com.thinkaurelius.titan.diskstorage.keycolumnvalue.Entry; @@ -12,6 +14,7 @@ import com.thinkaurelius.titan.diskstorage.keycolumnvalue.StoreFeatures; import com.thinkaurelius.titan.diskstorage.keycolumnvalue.StoreTransaction; import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration; +import java.io.File; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.*; @@ -69,6 +72,7 @@ public class AccumuloStoreManager extends DistributedStoreManager implements Key private final Connector connector; private final ConcurrentMap openStores; private final StoreFeatures features; + private final AccumuloStorageConfiguration storageConfig; public AccumuloStoreManager(Configuration config) throws StorageException { super(config, PORT_DEFAULT); @@ -106,6 +110,8 @@ public AccumuloStoreManager(Configuration config) throws StorageException { features.isKeyOrdered = false; features.isDistributed = true; features.hasLocalKeyPartition = false; + + storageConfig = new AccumuloStorageConfiguration(connector, tableName); } protected Instance createInstance(String instanceName, String zooKeepers) { @@ -308,54 +314,13 @@ public void clearStorage() throws StorageException { private ConcurrentMap properties; @Override - public String getConfigurationProperty(final String key) throws StorageException { - ensureTableExists(tableName); - - if (properties == null) { - properties = new ConcurrentHashMap(); - } - - return properties.get(key); - - /* - - TableOperations operations = connector.tableOperations(); - try { - Iterable> propIterator = operations.getProperties(tableName); - - Map properties = new HashMap(); - for (Map.Entry entry : propIterator) { - properties.put(entry.getKey(), entry.getValue()); - } - return properties.get(key); - } catch (AccumuloException ex) { - logger.error(ex.getMessage(), ex); - throw new PermanentStorageException(ex); - } catch (TableNotFoundException ex) { - logger.error(ex.getMessage(), ex); - throw new PermanentStorageException(ex); - } - */ + public String getConfigurationProperty(String key) throws StorageException { + return storageConfig.getConfigurationProperty(key); } @Override - public void setConfigurationProperty(final String key, final String value) throws StorageException { - ensureTableExists(tableName); - - properties.put(key, value); - /* - - TableOperations operations = connector.tableOperations(); - try { - operations.setProperty(tableName, key, value); - } catch (AccumuloException ex) { - logger.error(ex.getMessage(), ex); - throw new PermanentStorageException(ex); - } catch (AccumuloSecurityException ex) { - logger.error(ex.getMessage(), ex); - throw new PermanentStorageException(ex); - } - */ + public void setConfigurationProperty(String key, String value) throws StorageException { + storageConfig.setConfigurationProperty(key,value); } private static void waitUntil(long until) { diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/util/AccumuloStorageConfiguration.java b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/util/AccumuloStorageConfiguration.java new file mode 100644 index 0000000000..eee0538678 --- /dev/null +++ b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/util/AccumuloStorageConfiguration.java @@ -0,0 +1,98 @@ +package com.thinkaurelius.titan.diskstorage.accumulo.util; + +import com.google.common.base.Preconditions; +import com.thinkaurelius.titan.diskstorage.PermanentStorageException; +import com.thinkaurelius.titan.diskstorage.StorageException; +import com.thinkaurelius.titan.diskstorage.TemporaryStorageException; +import com.thinkaurelius.titan.diskstorage.accumulo.AccumuloStoreManager; +import org.apache.commons.configuration.Configuration; +import org.apache.commons.configuration.ConfigurationException; +import org.apache.commons.configuration.PropertiesConfiguration; +import java.io.File; +import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.MutationsRejectedException; +import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.client.TableNotFoundException; +import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.Value; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.hadoop.io.Text; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Implementation of storage backend properties using a local configuration + * file. + * + * Each storage backend provides the functionality to get and set properties for + * that particular backend. This class implementation this feature using a local + * configuration file. Hence, it is only suitable for local storage backends. + * + * @author Matthias Broecheler (me@matthiasb.com) + */ +public class AccumuloStorageConfiguration { + + private static final Logger logger = LoggerFactory.getLogger(AccumuloStorageConfiguration.class); + private static final String COL_FAMILY_DEFAULT = "_properties"; + private static final String ROW_ID_DEFAULT = ""; + private static final Long MAX_MEMORY_DEFAULT = 50 * 1024 * 1024l; + private static final Long MAX_LATENCY_DEFAULT = 100l; + private static final Integer MAX_WRITE_THREADS_DEFAULT = 10; + private final Connector connector; + private final String tableName; + private final String colFamily; + + public AccumuloStorageConfiguration(Connector connector, String tableName) { + this(connector, tableName, COL_FAMILY_DEFAULT); + } + + public AccumuloStorageConfiguration(Connector connector, String tableName, String colFamily) { + Preconditions.checkNotNull(connector); + Preconditions.checkNotNull(tableName); + Preconditions.checkArgument(tableName.length() > 0, "Table name is empty"); + this.connector = connector; + this.tableName = tableName; + this.colFamily = colFamily; + } + + public String getConfigurationProperty(String key) throws StorageException { + try { + Scanner scanner = connector.createScanner(tableName, new Authorizations()); + + Key scanKey = new Key(new Text(ROW_ID_DEFAULT), new Text(colFamily), new Text(key)); + scanner.setRange(new Range(key)); + + if (scanner.iterator().hasNext()) { + return scanner.iterator().next().getValue().toString(); + } else { + return null; + } + } catch (TableNotFoundException ex) { + logger.error("Can't find storage table " + tableName, ex); + throw new PermanentStorageException(ex); + } + } + + public void setConfigurationProperty(String key, String value) throws StorageException { + try { + BatchWriter writer = connector.createBatchWriter(tableName, + MAX_MEMORY_DEFAULT, MAX_LATENCY_DEFAULT, MAX_WRITE_THREADS_DEFAULT); + + Mutation mutation = new Mutation(ROW_ID_DEFAULT); + mutation.put(COL_FAMILY_DEFAULT, key, new Value(value.getBytes())); + + writer.addMutation(mutation); + + writer.close(); + } catch (TableNotFoundException ex) { + logger.error("Can't find storage table " + tableName, ex); + throw new PermanentStorageException(ex); + } catch (MutationsRejectedException ex) { + logger.error("Can't set configuration property " + tableName, ex); + throw new TemporaryStorageException(ex); + } + } +} diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/MutablePair.java b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/util/MutablePair.java similarity index 97% rename from titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/MutablePair.java rename to titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/util/MutablePair.java index 747b0422d1..d065e3fba2 100644 --- a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/MutablePair.java +++ b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/util/MutablePair.java @@ -2,7 +2,7 @@ * To change this template, choose Tools | Templates * and open the template in the editor. */ -package com.thinkaurelius.titan.diskstorage.accumulo; +package com.thinkaurelius.titan.diskstorage.accumulo.util; /** * A generic class for pairs. From a10e4767c33cfb0435fe797b115cceeae8dffd0a Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Fri, 21 Jun 2013 13:37:38 -0400 Subject: [PATCH 17/50] Light cleanup. --- .../accumulo/util/AccumuloStorageConfiguration.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/util/AccumuloStorageConfiguration.java b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/util/AccumuloStorageConfiguration.java index eee0538678..cd66092121 100644 --- a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/util/AccumuloStorageConfiguration.java +++ b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/util/AccumuloStorageConfiguration.java @@ -4,11 +4,6 @@ import com.thinkaurelius.titan.diskstorage.PermanentStorageException; import com.thinkaurelius.titan.diskstorage.StorageException; import com.thinkaurelius.titan.diskstorage.TemporaryStorageException; -import com.thinkaurelius.titan.diskstorage.accumulo.AccumuloStoreManager; -import org.apache.commons.configuration.Configuration; -import org.apache.commons.configuration.ConfigurationException; -import org.apache.commons.configuration.PropertiesConfiguration; -import java.io.File; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.Connector; import org.apache.accumulo.core.client.MutationsRejectedException; From 64940d19c09772ab311670a41a3bf16251dba684 Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Sun, 23 Jun 2013 21:19:53 -0400 Subject: [PATCH 18/50] Cleanup package imports. --- .../diskstorage/accumulo/AccumuloStoreManager.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java index fb32ec7096..16018a7d5d 100644 --- a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java +++ b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java @@ -14,10 +14,11 @@ import com.thinkaurelius.titan.diskstorage.keycolumnvalue.StoreFeatures; import com.thinkaurelius.titan.diskstorage.keycolumnvalue.StoreTransaction; import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration; -import java.io.File; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import org.apache.accumulo.core.client.AccumuloException; @@ -37,6 +38,9 @@ import org.apache.accumulo.core.security.Authorizations; import org.apache.commons.configuration.Configuration; import org.apache.hadoop.io.Text; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + /** * Storage Manager for HBase From 1aad34c90d0f422c5ee32dfb4e2622bbb378ed16 Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Fri, 28 Jun 2013 15:47:55 -0400 Subject: [PATCH 19/50] Cleaned up GraphOfTheGods example. --- .../accumulo/GraphOfTheGodsFactory.java | 167 ++++++++++++++---- 1 file changed, 131 insertions(+), 36 deletions(-) diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/example/accumulo/GraphOfTheGodsFactory.java b/titan-accumulo/src/main/java/com/thinkaurelius/titan/example/accumulo/GraphOfTheGodsFactory.java index 1e183a7acc..10f1534012 100644 --- a/titan-accumulo/src/main/java/com/thinkaurelius/titan/example/accumulo/GraphOfTheGodsFactory.java +++ b/titan-accumulo/src/main/java/com/thinkaurelius/titan/example/accumulo/GraphOfTheGodsFactory.java @@ -12,8 +12,19 @@ import com.tinkerpop.blueprints.util.ElementHelper; import org.apache.commons.configuration.BaseConfiguration; import org.apache.commons.configuration.Configuration; -import java.io.File; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.OptionBuilder; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.commons.cli.PosixParser; +import org.slf4j.LoggerFactory; /** * Example Graph factory that creates a {@link TitanGraph} based on roman @@ -23,27 +34,30 @@ */ public class GraphOfTheGodsFactory { - public static final String INDEX_NAME = "search"; + private static final org.slf4j.Logger logger = LoggerFactory.getLogger(GraphOfTheGodsFactory.class); + private static final String ZOOKEEPERS_DEFAULT = "localhost"; + private static final String USERNAME_DEFAULT = "root"; + private String instance; + private String zooKeepers; + private String username; + private String password; - public static void main(String[] args) throws IOException { - TitanGraph graph; + public TitanGraph create() { + return create(instance, zooKeepers, username, password); + } - System.out.println("Creating graph ... "); - graph = open(args[0]); - System.out.println("Loading graph ... "); - load(graph); - System.out.println("Done!"); + public static TitanGraph create(final String instance, final String zooKeepers, + final String username, final String password) { + + return load(define(open(instance, zooKeepers, username, password))); } - - public static TitanGraph create(final String directory) { - TitanGraph graph; - - graph = open(directory); - load(graph); - return graph; + + public TitanGraph open() { + return open(instance, zooKeepers, username, password); } - public static TitanGraph open(final String directory) { + public static TitanGraph open(final String instance, final String zooKeepers, + final String username, final String password) { BaseConfiguration config = new BaseConfiguration(); Configuration storage = config.subset(GraphDatabaseConfiguration.STORAGE_NAMESPACE); // configuring local backend @@ -53,34 +67,26 @@ public static TitanGraph open(final String directory) { Configuration accumulo = storage.subset(AccumuloStoreManager.ACCUMULO_CONFIGURATION_NAMESPACE); - accumulo.addProperty(AccumuloStoreManager.ACCUMULO_INTSANCE_KEY, "EtCloud"); + accumulo.addProperty(AccumuloStoreManager.ACCUMULO_INTSANCE_KEY, instance); accumulo.addProperty(GraphDatabaseConfiguration.HOSTNAME_KEY, "localhost"); - accumulo.addProperty(AccumuloStoreManager.ACCUMULO_USER_KEY, "root"); - accumulo.addProperty(AccumuloStoreManager.ACCUMULO_PASSWORD_KEY, "bobross"); - // configuring elastic search index - Configuration index = storage.subset(GraphDatabaseConfiguration.INDEX_NAMESPACE).subset(INDEX_NAME); - index.setProperty(GraphDatabaseConfiguration.INDEX_BACKEND_KEY, "lucene"); - /* - index.setProperty("local-mode", true); - index.setProperty("client-only", false); - */ - index.setProperty(GraphDatabaseConfiguration.STORAGE_DIRECTORY_KEY, directory + File.separator + "lucene"); + accumulo.addProperty(AccumuloStoreManager.ACCUMULO_USER_KEY, username); + accumulo.addProperty(AccumuloStoreManager.ACCUMULO_PASSWORD_KEY, password); TitanGraph graph = TitanFactory.open(config); return graph; } - public static void load(final TitanGraph graph) { - + public static TitanGraph define(final TitanGraph graph) { + // Define types graph.makeType().name("name").dataType(String.class).indexed(Vertex.class).unique(Direction.BOTH).makePropertyKey(); - graph.makeType().name("age").dataType(Integer.class).indexed(INDEX_NAME, Vertex.class).unique(Direction.OUT).makePropertyKey(); + graph.makeType().name("age").dataType(Integer.class).indexed(Vertex.class).unique(Direction.OUT).makePropertyKey(); graph.makeType().name("type").dataType(String.class).unique(Direction.OUT).makePropertyKey(); final TitanKey time = graph.makeType().name("time").dataType(Integer.class).unique(Direction.OUT).makePropertyKey(); - final TitanKey reason = graph.makeType().name("reason").dataType(String.class).indexed(INDEX_NAME, Edge.class).unique(Direction.OUT).makePropertyKey(); - graph.makeType().name("place").dataType(Geoshape.class).indexed(INDEX_NAME, Edge.class).unique(Direction.OUT).makePropertyKey(); + final TitanKey reason = graph.makeType().name("reason").dataType(String.class).indexed(Edge.class).unique(Direction.OUT).makePropertyKey(); + graph.makeType().name("place").dataType(Geoshape.class).indexed(Edge.class).unique(Direction.OUT).makePropertyKey(); graph.makeType().name("father").unique(Direction.OUT).makeEdgeLabel(); graph.makeType().name("mother").unique(Direction.OUT).makeEdgeLabel(); @@ -91,8 +97,12 @@ public static void load(final TitanGraph graph) { graph.commit(); - // vertices + return graph; + } + + public static TitanGraph load(final TitanGraph graph) { + // vertices Vertex saturn = graph.addVertex(null); saturn.setProperty("name", "saturn"); saturn.setProperty("age", 10000); @@ -132,7 +142,6 @@ public static void load(final TitanGraph graph) { ElementHelper.setProperties(tartarus, "name", "tartarus", "type", "location"); // edges - jupiter.addEdge("father", saturn); jupiter.addEdge("lives", sky).setProperty("reason", "loves fresh breezes"); jupiter.addEdge("brother", neptune); @@ -155,7 +164,93 @@ public static void load(final TitanGraph graph) { cerberus.addEdge("lives", tartarus); - // commit the transaction to disk graph.commit(); + + return graph; + } + + private static Collection

+ * Default: 3 + * + * @param numQueryThreads the number threads to use + * @throws IllegalArgumentException if {@code maxWriteThreads} is + * non-positive + * @return {@code this} to allow chaining of set methods + */ + public AccumuloBatchConfiguration setNumQueryThreads(int numQueryThreads) { + if (numQueryThreads <= 0) { + throw new IllegalArgumentException("Num threads must be positive " + numQueryThreads); + } + + this.numQueryThreads = numQueryThreads; + return this; + } + + /** + * Sets the maximum memory to batch before writing. The smaller this value, + * the more frequently the {@link BatchWriter} will write.
+ * If set to a value smaller than a single mutation, then it will + * {@link BatchWriter#flush()} after each added mutation. Must be + * non-negative. + * + *

+ * Default: 50M + * + * @param maxMemory max size in bytes + * @throws IllegalArgumentException if {@code maxMemory} is less than 0 + * @return {@code this} to allow chaining of set methods + */ + public AccumuloBatchConfiguration setMaxMemory(long maxMemory) { + if (maxMemory < 0) { + throw new IllegalArgumentException("Max memory must be non-negative."); + } + this.maxMemory = maxMemory; + return this; + } + + /** + * Sets the maximum amount of time to hold the data in memory before + * flushing it to servers.
+ * For no maximum, set to zero, or {@link Long#MAX_VALUE} with + * {@link TimeUnit#MILLISECONDS}. + * + *

{@link TimeUnit#MICROSECONDS} or {@link TimeUnit#NANOSECONDS} will be + * truncated to the nearest {@link TimeUnit#MILLISECONDS}.
+ * If this truncation would result in making the value zero when it was + * specified as non-zero, then a minimum value of one + * {@link TimeUnit#MILLISECONDS} will be used. + * + *

+ * Default: 120 seconds + * + * @param maxLatency the maximum latency, in the unit specified by the value + * of {@code timeUnit} + * @param timeUnit determines how {@code maxLatency} will be interpreted + * @throws IllegalArgumentException if {@code maxLatency} is less than 0 + * @return {@code this} to allow chaining of set methods + */ + public AccumuloBatchConfiguration setMaxLatency(long maxLatency, TimeUnit timeUnit) { + if (maxLatency < 0) { + throw new IllegalArgumentException("Negative max latency not allowed " + maxLatency); + } + + if (maxLatency == 0) { + this.maxLatency = Long.MAX_VALUE; + } else { // make small, positive values that truncate to 0 when converted use the minimum millis instead + this.maxLatency = Math.max(1, timeUnit.toMillis(maxLatency)); + } + return this; + } + + /** + * Sets the maximum amount of time an unresponsive server will be re-tried. + * When this timeout is exceeded, the {@link BatchWriter} should throw an + * exception.
+ * For no timeout, set to zero, or {@link Long#MAX_VALUE} with + * {@link TimeUnit#MILLISECONDS}. + * + *

{@link TimeUnit#MICROSECONDS} or {@link TimeUnit#NANOSECONDS} will be + * truncated to the nearest {@link TimeUnit#MILLISECONDS}.
+ * If this truncation would result in making the value zero when it was + * specified as non-zero, then a minimum value of one + * {@link TimeUnit#MILLISECONDS} will be used. + * + *

+ * Default: {@link Long#MAX_VALUE} (no timeout) + * + * @param timeout the timeout, in the unit specified by the value of + * {@code timeUnit} + * @param timeUnit determines how {@code timeout} will be interpreted + * @throws IllegalArgumentException if {@code timeout} is less than 0 + * @return {@code this} to allow chaining of set methods + */ + public AccumuloBatchConfiguration setTimeout(long timeout, TimeUnit timeUnit) { + if (timeout < 0) { + throw new IllegalArgumentException("Negative timeout not allowed " + timeout); + } + + if (timeout == 0) { + this.timeout = Long.MAX_VALUE; + } else { // make small, positive values that truncate to 0 when converted use the minimum millis instead + this.timeout = Math.max(1, timeUnit.toMillis(timeout)); + } + return this; + } + + /** + * Sets the maximum number of threads to use for writing data to the tablet + * servers. + * + *

+ * Default: 3 + * + * @param maxWriteThreads the maximum threads to use + * @throws IllegalArgumentException if {@code maxWriteThreads} is + * non-positive + * @return {@code this} to allow chaining of set methods + */ + public AccumuloBatchConfiguration setMaxWriteThreads(int maxWriteThreads) { + if (maxWriteThreads <= 0) { + throw new IllegalArgumentException("Max threads must be positive " + maxWriteThreads); + } + + this.maxWriteThreads = maxWriteThreads; + return this; + } + + /** + * Get number of threads to spawn for querying on tablet servers. + * + * @return number of threads + */ + public int getNumQueryThreads() { + return numQueryThreads != null ? numQueryThreads : DEFAULT_NUM_QUERY_THREADS; + } + + /** + * Sets the maximum memory to batch before writing. + * + * @return max memory + */ + public long getMaxMemory() { + return maxMemory != null ? maxMemory : DEFAULT_MAX_MEMORY; + } + + /** + * Sets the maximum amount of time to hold the data in memory before flushing it to servers. + * + * @param timeUnit units for return value + * @return max latency + */ + public long getMaxLatency(TimeUnit timeUnit) { + return timeUnit.convert(maxLatency != null ? maxLatency : DEFAULT_MAX_LATENCY, TimeUnit.MILLISECONDS); + } + + /** + * Get the maximum amount of time an unresponsive server will be re-tried. + * + * @param timeUnit units for return value + * @return max timeout + */ + public long getTimeout(TimeUnit timeUnit) { + return timeUnit.convert(timeout != null ? timeout : DEFAULT_TIMEOUT, TimeUnit.MILLISECONDS); + } + + /** + * Get the maximum number of threads to use for writing data to the tablet servers. + * + * @return max threads + */ + public int getMaxWriteThreads() { + return maxWriteThreads != null ? maxWriteThreads : DEFAULT_MAX_WRITE_THREADS; + } + + /** + * Factory method to create a BatchDeleter connected to Accumulo. + * + * @param connector connection to Accumulo + * @param tableName the name of the table to query and delete from + * @param authorizations set of authorization labels that will be checked against the column visibility. + * @return BatchDeleter object for configuring and deleting + * @throws TableNotFoundException when the specified table doesn't exist + */ + public BatchDeleter createBatchDeleter(Connector connector, String tableName, Authorizations authorizations) throws TableNotFoundException { + return connector.createBatchDeleter(tableName, authorizations, + getNumQueryThreads(), getMaxMemory(), getMaxLatency(TimeUnit.MILLISECONDS), getMaxWriteThreads()); + } + + /** + * Factory method to create a BatchScanner connected to Accumulo. + * + * @param connector connection to Accumulo + * @param tableName the name of the table to query and delete from + * @param authorizations set of authorization labels that will be checked against the column visibility. + * @return BatchScanner object for configuring and querying + * @throws TableNotFoundException when the specified table doesn't exist + */ + public BatchScanner createBatchScanner(Connector connector, String tableName, Authorizations authorizations) throws TableNotFoundException { + return connector.createBatchScanner(tableName, authorizations, getNumQueryThreads()); + } + + /** + * Factory method to create a BatchWriter connected to Accumulo. + * + * @param connector connection to Accumulo + * @param tableName the name of the table to insert data into + * @return BatchWriter object for configuring and writing data + * @throws TableNotFoundException when the specified table doesn't exist + */ + public BatchWriter createBatchWriter(Connector connector, String tableName) throws TableNotFoundException { + return connector.createBatchWriter(tableName, + getMaxMemory(), getMaxLatency(TimeUnit.MILLISECONDS), getMaxWriteThreads()); + } +} \ No newline at end of file diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloInstanceInjector.java b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloInstanceFactory.java similarity index 68% rename from titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloInstanceInjector.java rename to titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloInstanceFactory.java index 2fd745f2ea..71cba81a3a 100644 --- a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloInstanceInjector.java +++ b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloInstanceFactory.java @@ -4,16 +4,16 @@ import org.apache.accumulo.core.client.ZooKeeperInstance; /** - * Instance injector for Accumulo store configuration. + * Instance factory for Accumulo store configuration. * * @author Etienne Deprit */ -public interface AccumuloInstanceInjector { +public interface AccumuloInstanceFactory { public Instance getInstance(String instanceName, String zooKeepers); // - public static final AccumuloInstanceInjector ZOOKEEPER_INSTANCE_INJECTOR = - new AccumuloInstanceInjector() { + public static final AccumuloInstanceFactory ZOOKEEPER_INSTANCE_FACTORY = + new AccumuloInstanceFactory() { @Override public Instance getInstance(String instanceName, String zooKeepers) { return new ZooKeeperInstance(instanceName, zooKeepers); diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java index c9421b93dd..5d9ebb3948 100644 --- a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java +++ b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java @@ -1,7 +1,6 @@ package com.thinkaurelius.titan.diskstorage.accumulo; import com.google.common.base.Preconditions; -import com.google.common.base.Predicate; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -25,8 +24,6 @@ import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.logging.Level; -import javax.annotation.Nullable; import org.apache.accumulo.core.client.BatchScanner; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.ClientSideIteratorScanner; @@ -48,31 +45,15 @@ import org.apache.hadoop.io.Text; /** - * Experimental Accumulo store. - *

- * This is not ready for production. It's pretty slow. - *

- * Here are some areas that might need work: - *

- * - batching? (consider HTable#batch, HTable#setAutoFlush(false) - tuning - * HTable#setWriteBufferSize (?) - writing a server-side filter to replace - * ColumnCountGetFilter, which drops all columns on the row where it reaches its - * limit. This requires getSlice, currently, to impose its limit on the client - * side. That obviously won't scale. - RowMutations for combining Puts+Deletes - * (need a newer HBase than 0.92 for this) - (maybe) fiddle with - * HTable#setRegionCachePrefetch and/or #prewarmRegionCache - *

- * There may be other problem areas. These are just the ones of which I'm aware. + * Key-Column value store for Accumulo + * + * @author Etienne Deprit */ public class AccumuloKeyColumnValueStore implements KeyColumnValueStore { private static final Logger logger = LoggerFactory.getLogger(AccumuloKeyColumnValueStore.class); // Default deleter, scanner, writer parameters - private static final Authorizations DEFAULT_AUTHORIZATIONS = new Authorizations(); - private static final Long DEFAULT_MAX_MEMORY = 50 * 1024 * 1024l; - private static final Long DEFAULT_MAX_LATENCY = 100l; - private static final Integer DEFAULT_MAX_QUERY_THREADS = 10; - private static final Integer DEFAULT_MAX_WRITE_THREADS = 10; + private static final Authorizations AUTHORIZATIONS_DEFAULT = new Authorizations(); // Instance variables private final Connector connector; // thread-safe private final String tableName; @@ -80,14 +61,17 @@ public class AccumuloKeyColumnValueStore implements KeyColumnValueStore { // This is columnFamily.getBytes() private final byte[] columnFamilyBytes; private final Text columnFamilyText; + private final AccumuloBatchConfiguration batchConfiguration; private final boolean clientSideIterators; - AccumuloKeyColumnValueStore(Connector connector, String tableName, String columnFamily, boolean clientSideIterators) { + AccumuloKeyColumnValueStore(Connector connector, String tableName, String columnFamily, + AccumuloBatchConfiguration batchConfiguration, boolean clientSideIterators) { this.connector = connector; this.tableName = tableName; this.columnFamily = columnFamily; this.columnFamilyBytes = columnFamily.getBytes(); this.columnFamilyText = new Text(columnFamily); + this.batchConfiguration = batchConfiguration; this.clientSideIterators = clientSideIterators; } @@ -104,7 +88,7 @@ public void close() throws StorageException { public boolean containsKey(StaticBuffer key, StoreTransaction txh) throws StorageException { Scanner scanner; try { - scanner = connector.createScanner(tableName, DEFAULT_AUTHORIZATIONS); + scanner = connector.createScanner(tableName, AUTHORIZATIONS_DEFAULT); } catch (TableNotFoundException ex) { logger.error("Can't find Titan store " + tableName, ex); throw new PermanentStorageException(ex); @@ -121,32 +105,13 @@ public boolean containsKey(StaticBuffer key, StoreTransaction txh) throws Storag public List getSlice(KeySliceQuery query, StoreTransaction txh) throws StorageException { Scanner scanner; try { - scanner = connector.createScanner(tableName, DEFAULT_AUTHORIZATIONS); + scanner = connector.createScanner(tableName, AUTHORIZATIONS_DEFAULT); } catch (TableNotFoundException ex) { logger.error("Can't find Titan store " + tableName, ex); throw new PermanentStorageException(ex); } - Text keyText = new Text(query.getKey().as(StaticBuffer.ARRAY_FACTORY)); - - Key startKey; - Key endKey; - - if (query.getSliceStart().length() > 0) { - startKey = new Key(keyText, columnFamilyText, - new Text(query.getSliceStart().as(StaticBuffer.ARRAY_FACTORY))); - } else { - startKey = new Key(keyText, columnFamilyText); - } - - if (query.getSliceEnd().length() > 0) { - endKey = new Key(keyText, columnFamilyText, - new Text(query.getSliceEnd().as(StaticBuffer.ARRAY_FACTORY))); - } else { - endKey = new Key(keyText, columnFamilyText); - } - - scanner.setRange(new Range(startKey, true, endKey, false)); + scanner.setRange(getRange(query)); if (query.getLimit() < scanner.getBatchSize()) { scanner.setBatchSize(query.getLimit()); } @@ -171,7 +136,7 @@ public List getSlice(KeySliceQuery query, StoreTransaction txh) throws St public List> getSlice(List keys, SliceQuery query, StoreTransaction txh) throws StorageException { BatchScanner scanner; try { - scanner = connector.createBatchScanner(tableName, DEFAULT_AUTHORIZATIONS, DEFAULT_MAX_QUERY_THREADS); + scanner = batchConfiguration.createBatchScanner(connector, tableName, AUTHORIZATIONS_DEFAULT); } catch (TableNotFoundException ex) { logger.error("Can't find Titan store " + tableName, ex); throw new PermanentStorageException(ex); @@ -184,24 +149,7 @@ public List> getSlice(List keys, SliceQuery query, Sto Collection ranges = Lists.newArrayList(); for (Text key : keysText) { - Key startKey; - Key endKey; - - if (query.getSliceStart().length() > 0) { - startKey = new Key(key, columnFamilyText, - new Text(query.getSliceStart().as(StaticBuffer.ARRAY_FACTORY))); - } else { - startKey = new Key(key, columnFamilyText); - } - - if (query.getSliceEnd().length() > 0) { - endKey = new Key(key, columnFamilyText, - new Text(query.getSliceEnd().as(StaticBuffer.ARRAY_FACTORY))); - } else { - endKey = new Key(key, columnFamilyText); - } - - ranges.add(new Range(startKey, true, endKey, false)); + ranges.add(getRange(key, query)); } scanner.setRanges(ranges); @@ -247,8 +195,7 @@ public void mutate(StaticBuffer key, List additions, List d } try { - BatchWriter writer = connector.createBatchWriter(tableName, - DEFAULT_MAX_MEMORY, DEFAULT_MAX_LATENCY, DEFAULT_MAX_WRITE_THREADS); + BatchWriter writer = batchConfiguration.createBatchWriter(connector, tableName); try { writer.addMutations(batch); writer.flush(); @@ -342,86 +289,62 @@ public StaticBuffer[] getLocalKeyPartition() throws StorageException { @Override public RecordIterator getKeys(StoreTransaction txh) throws StorageException { - final BatchScanner scanner; - - try { - scanner = connector.createBatchScanner(tableName, - DEFAULT_AUTHORIZATIONS, DEFAULT_MAX_QUERY_THREADS); - } catch (TableNotFoundException ex) { - logger.error("Can't find Titan store " + tableName, ex); - throw new PermanentStorageException(ex); - } - - scanner.setRanges(Collections.singletonList(new Range())); - - scanner.fetchColumnFamily(new Text(columnFamily)); - - IteratorSetting firstRowKeyIterator = new IteratorSetting(10, "firstRowKeyIter", FirstEntryInRowIterator.class); - scanner.addScanIterator(firstRowKeyIterator); - - return new RecordIterator() { - private final Iterator> results = scanner.iterator(); - - @Override - public boolean hasNext() throws StorageException { - return results.hasNext(); - } - - @Override - public StaticBuffer next() throws StorageException { - return new StaticArrayBuffer(results.next().getKey().getRow().getBytes()); - } - - @Override - public void close() throws StorageException { - scanner.close(); - } - }; + return executeKeySliceQuery(null, true); } @Override public KeyIterator getKeys(KeyRangeQuery query, StoreTransaction txh) throws StorageException { - return executeKeySliceQuery(query.getKeyStart(), query.getKeyEnd(), query); + return executeKeySliceQuery(query.getKeyStart(), query.getKeyEnd(), query, false); } @Override public KeyIterator getKeys(SliceQuery query, StoreTransaction txh) throws StorageException { - return executeKeySliceQuery(null, null, query); + return executeKeySliceQuery(query, false); + } + + private KeyIterator executeKeySliceQuery(SliceQuery columnSlice, boolean keysOnly) throws StorageException { + return executeKeySliceQuery(null, null, columnSlice, keysOnly); } private KeyIterator executeKeySliceQuery(StaticBuffer startKey, StaticBuffer endKey, - SliceQuery columnSlice) throws StorageException { + SliceQuery columnSlice, boolean keysOnly) throws StorageException { - Scanner scanner = getKeySliceScanner(startKey, endKey, columnSlice); + Scanner scanner = getKeySliceScanner(startKey, endKey, columnSlice, keysOnly); return new RowKeyIterator(scanner); } - private Scanner getKeySliceScanner(StaticBuffer startKey, StaticBuffer endKey, SliceQuery columnSlice) throws StorageException { - Range range = getRange(startKey, endKey); + private Scanner getKeySliceScanner(StaticBuffer startKey, StaticBuffer endKey, + SliceQuery columnSlice, boolean keysOnly) throws StorageException { Scanner scanner; try { - scanner = connector.createScanner(tableName, DEFAULT_AUTHORIZATIONS); + scanner = connector.createScanner(tableName, AUTHORIZATIONS_DEFAULT); } catch (TableNotFoundException ex) { logger.error("Can't find Titan store " + tableName, ex); throw new PermanentStorageException(ex); } + Range range = getRange(startKey, endKey); scanner.setRange(range); scanner.fetchColumnFamily(columnFamilyText); IteratorSetting columnSliceIterator = null; if (columnSlice != null) { columnSliceIterator = getColumnSliceIterator(columnSlice); - } - if (columnSliceIterator != null) { - if (clientSideIterators) { - scanner = new ClientSideIteratorScanner(scanner); + if (columnSliceIterator != null) { + if (clientSideIterators) { + scanner = new ClientSideIteratorScanner(scanner); + } + scanner.addScanIterator(columnSliceIterator); } - scanner.addScanIterator(columnSliceIterator); + } + + if (keysOnly) { + IteratorSetting firstRowKeyIterator = new IteratorSetting(15, "firstRowKeyIter", FirstEntryInRowIterator.class); + scanner.addScanIterator(firstRowKeyIterator); } return scanner; @@ -442,6 +365,32 @@ private Range getRange(StaticBuffer startKey, StaticBuffer endKey) { return new Range(startRow, true, endRow, false); } + private Range getRange(KeySliceQuery query) { + Text key = new Text(query.getKey().as(StaticBuffer.ARRAY_FACTORY)); + return getRange(key, query); + } + + private Range getRange(Text key, SliceQuery query) { + Key startKey; + Key endKey; + + if (query.getSliceStart().length() > 0) { + startKey = new Key(key, columnFamilyText, + new Text(query.getSliceStart().as(StaticBuffer.ARRAY_FACTORY))); + } else { + startKey = new Key(key, columnFamilyText); + } + + if (query.getSliceEnd().length() > 0) { + endKey = new Key(key, columnFamilyText, + new Text(query.getSliceEnd().as(StaticBuffer.ARRAY_FACTORY))); + } else { + endKey = new Key(key, columnFamilyText); + } + + return new Range(startKey, true, endKey, false); + } + private IteratorSetting getColumnSliceIterator(SliceQuery sliceQuery) { IteratorSetting is = null; @@ -449,14 +398,14 @@ private IteratorSetting getColumnSliceIterator(SliceQuery sliceQuery) { byte[] maxColumn = sliceQuery.getSliceEnd().as(StaticBuffer.ARRAY_FACTORY); if (minColumn.length > 0 || maxColumn.length > 0) { - is = new IteratorSetting(5, "columnRangeIter", ColumnRangeFilter.class); + is = new IteratorSetting(10, "columnRangeIter", ColumnRangeFilter.class); ColumnRangeFilter.setRange(is, minColumn, true, maxColumn, false); } return is; } - private class RowKeyIterator implements KeyIterator { + private static class RowKeyIterator implements KeyIterator { RowIterator rows; PeekingIterator> currentRow; @@ -470,8 +419,6 @@ private class RowKeyIterator implements KeyIterator { @Override public RecordIterator getEntries() { RecordIterator rowIter = new RecordIterator() { - boolean isClosed = false; - @Override public boolean hasNext() throws StorageException { ensureOpen(); @@ -492,14 +439,7 @@ public Entry next() throws StorageException { @Override public void close() throws StorageException { - isClosed = true; - currentRow = null; - } - - private void ensureOpen() { - if (isClosed) { - throw new IllegalStateException("Iterator has been closed."); - } + isClosed = true; // same semantics as in-memory implementation in Titan core } }; diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreConfiguration.java b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreConfiguration.java index 433ed610cc..d686305ff8 100644 --- a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreConfiguration.java +++ b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreConfiguration.java @@ -33,26 +33,26 @@ public class AccumuloStoreConfiguration { private static final Logger logger = LoggerFactory.getLogger(AccumuloStoreConfiguration.class); - // Default deleter, scanner, writer parameters - private static final Long MAX_MEMORY_DEFAULT = 50 * 1024 * 1024l; - private static final Long MAX_LATENCY_DEFAULT = 100l; - private static final Integer MAX_WRITE_THREADS_DEFAULT = 10; // Configuration defaults private static final String ROW_ID_DEFAULT = ""; private static final String COL_FAMILY_DEFAULT = "_properties"; + private static final Authorizations AUTHORIZATIONS_DEFAULT = new Authorizations(); // Instance variables private final Connector connector; // thread-safe private final String tableName; + private final AccumuloBatchConfiguration batchConfiguration; private final Text rowIdText; private final Text colFamilyText; - public AccumuloStoreConfiguration(Connector connector, String tableName) { - this(connector, tableName, ROW_ID_DEFAULT, COL_FAMILY_DEFAULT); + public AccumuloStoreConfiguration(Connector connector, String tableName, AccumuloBatchConfiguration batchConfiguration) { + this(connector, tableName, batchConfiguration, ROW_ID_DEFAULT, COL_FAMILY_DEFAULT); } - public AccumuloStoreConfiguration(Connector connector, String tableName, String rowId, String colFamily) { + public AccumuloStoreConfiguration(Connector connector, String tableName, + AccumuloBatchConfiguration batchConfiguration, String rowId, String colFamily) { this.connector = connector; this.tableName = tableName; + this.batchConfiguration = batchConfiguration; this.rowIdText = new Text(rowId); this.colFamilyText = new Text(colFamily); } @@ -62,7 +62,7 @@ public String getConfigurationProperty(String key) throws StorageException { Scanner scanner; try { - scanner = connector.createScanner(tableName, new Authorizations()); + scanner = connector.createScanner(tableName, AUTHORIZATIONS_DEFAULT); } catch (TableNotFoundException ex) { logger.error("Can't find Titan table " + tableName, ex); throw new PermanentStorageException(ex); @@ -82,9 +82,7 @@ public String getConfigurationProperty(String key) throws StorageException { public void setConfigurationProperty(String key, String value) throws StorageException { try { - BatchWriter writer = connector.createBatchWriter(tableName, - MAX_MEMORY_DEFAULT, MAX_LATENCY_DEFAULT, MAX_WRITE_THREADS_DEFAULT); - + BatchWriter writer = batchConfiguration.createBatchWriter(connector, tableName); try { Mutation mutation = new Mutation(rowIdText); mutation.put(colFamilyText, new Text(key), new Value(value.getBytes())); diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java index 1ce5919f2e..69ba3ab0bc 100644 --- a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java +++ b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java @@ -46,11 +46,7 @@ public class AccumuloStoreManager extends DistributedStoreManager implements Key private static final Logger logger = LoggerFactory.getLogger(AccumuloStoreManager.class); // Default deleter, scanner, writer parameters - private static final Authorizations DEFAULT_AUTHORIZATIONS = new Authorizations(); - private static final Long DEFAULT_MAX_MEMORY = 50 * 1024 * 1024l; - private static final Long DEFAULT_MAX_LATENCY = 100l; - private static final Integer DEFAULT_MAX_QUERY_THREADS = 10; - private static final Integer DEFAULT_MAX_WRITE_THREADS = 10; + private static final Authorizations AUTHORIZATIONS_DEFAULT = new Authorizations(); // Configuration namespace public static final String ACCUMULO_CONFIGURATION_NAMESPACE = "accumulo-config"; // Configuration keys @@ -64,7 +60,7 @@ public class AccumuloStoreManager extends DistributedStoreManager implements Key public static final int PORT_DEFAULT = 9160; public static final boolean CLIENT_SIDE_ITERATORS_DEFAULT = true; // Instance injector - public static AccumuloInstanceInjector instanceInjector = AccumuloInstanceInjector.ZOOKEEPER_INSTANCE_INJECTOR; + public static AccumuloInstanceFactory instanceFactory = AccumuloInstanceFactory.ZOOKEEPER_INSTANCE_FACTORY; // Instance variables private final String tableName; private final String instanceName; @@ -75,8 +71,9 @@ public class AccumuloStoreManager extends DistributedStoreManager implements Key private final Instance instance; // thread-safe private final Connector connector; // thread-safe private final ConcurrentMap openStores; - private final StoreFeatures features; // immutable once created - private final AccumuloStoreConfiguration storeConfiguration; + private final StoreFeatures features; // immutable at constructor exit + private final AccumuloBatchConfiguration batchConfiguration; // immutable at constructor exit + private final AccumuloStoreConfiguration storeConfiguration; // immutable at constructor exit public AccumuloStoreManager(Configuration config) throws StorageException { super(config, PORT_DEFAULT); @@ -94,7 +91,7 @@ public AccumuloStoreManager(Configuration config) throws StorageException { clientSideIterators = accumuloConfig.getBoolean(CLIENT_SIDE_ITERATORS_KEY, CLIENT_SIDE_ITERATORS_DEFAULT); - instance = instanceInjector.getInstance(instanceName, zooKeepers); + instance = instanceFactory.getInstance(instanceName, zooKeepers); try { connector = instance.getConnector(username, password.getBytes()); @@ -118,7 +115,9 @@ public AccumuloStoreManager(Configuration config) throws StorageException { features.isDistributed = true; features.hasLocalKeyPartition = false; - storeConfiguration = new AccumuloStoreConfiguration(connector, tableName); + batchConfiguration = new AccumuloBatchConfiguration(); + + storeConfiguration = new AccumuloStoreConfiguration(connector, tableName, batchConfiguration); } @Override @@ -146,7 +145,8 @@ public KeyColumnValueStore openDatabase(String dbName) throws StorageException { AccumuloKeyColumnValueStore store = openStores.get(dbName); if (store == null) { - AccumuloKeyColumnValueStore newStore = new AccumuloKeyColumnValueStore(connector, tableName, dbName, clientSideIterators); + AccumuloKeyColumnValueStore newStore = new AccumuloKeyColumnValueStore(connector, tableName, dbName, + batchConfiguration, clientSideIterators); store = openStores.putIfAbsent(dbName, newStore); // atomic so only one store dbName @@ -155,7 +155,7 @@ public KeyColumnValueStore openDatabase(String dbName) throws StorageException { store = newStore; } } - + return store; } @@ -167,8 +167,7 @@ public void mutateMany(Map> mutations, St Collection actions = convertToActions(mutations, putTS, delTS); try { - BatchWriter writer = connector.createBatchWriter(tableName, - DEFAULT_MAX_MEMORY, DEFAULT_MAX_LATENCY, DEFAULT_MAX_WRITE_THREADS); + BatchWriter writer = batchConfiguration.createBatchWriter(connector, tableName); try { writer.addMutations(actions); writer.flush(); @@ -280,8 +279,7 @@ public void clearStorage() throws StorageException { } try { - BatchDeleter deleter = connector.createBatchDeleter(tableName, DEFAULT_AUTHORIZATIONS, - DEFAULT_MAX_QUERY_THREADS, DEFAULT_MAX_MEMORY, DEFAULT_MAX_LATENCY, DEFAULT_MAX_WRITE_THREADS); + BatchDeleter deleter = batchConfiguration.createBatchDeleter(connector, tableName, AUTHORIZATIONS_DEFAULT); deleter.setRanges(Collections.singletonList(new Range())); try { deleter.delete(); diff --git a/titan-accumulo/src/main/resources/log4j.properties b/titan-accumulo/src/main/resources/log4j.properties index 9b72c91828..a02cd1ff42 100644 --- a/titan-accumulo/src/main/resources/log4j.properties +++ b/titan-accumulo/src/main/resources/log4j.properties @@ -8,6 +8,7 @@ log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n # Set root logger level to the designated level and its only appender to A1. log4j.rootLogger=INFO, A1 +log4j.logger.org.apache.accumulo=INFO log4j.logger.org.apache.cassandra=INFO log4j.logger.org.apache.hadoop=INFO log4j.logger.org.apache.zookeeper=INFO diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java index 4b7e932e7d..b80e6e216f 100644 --- a/titan-accumulo/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java +++ b/titan-accumulo/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java @@ -6,55 +6,11 @@ import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration; import org.apache.commons.configuration.BaseConfiguration; import org.apache.commons.configuration.Configuration; -import org.apache.commons.io.FileUtils; - -import java.io.File; import java.io.IOException; import java.lang.reflect.Constructor; public class AccumuloStorageSetup { - private static Process ACCUMULO = null; - // amount of seconds to wait before assuming that HBase shutdown - private static final int SHUTDOWN_TIMEOUT_SEC = 20; - // hbase config for testing - private static final String ACCUMULO_CONFIG_DIR = "./src/test/config"; - // default pid file location - private static final String ACCUMULO_PID_FILE = "/tmp/accumulo-" + System.getProperty("user.name") + "-master.pid"; - - static { - try { - System.out.println("Deleteing old test directories (if any)."); - - // please keep in sync with HBASE_CONFIG_DIR/hbase-site.xml, reading HBase XML config is huge pain. - File accumuloRoot = new File("./src/test/titan-accumulo-test-data"); - File zookeeperDataDir = new File("./src/test/titan-zookeeper-test"); - - if (accumuloRoot.exists()) { - FileUtils.deleteDirectory(accumuloRoot); - } - - if (zookeeperDataDir.exists()) { - FileUtils.deleteDirectory(zookeeperDataDir); - } - } catch (IOException e) { - System.err.println("Failed to delete old Accumulo test directories: '" + e.getMessage() + "', ignoring."); - } - - Runtime.getRuntime().addShutdownHook(new Thread() { - @Override - public void run() { - System.out.println("All done. Shutting done Accumulo."); - - try { - AccumuloStorageSetup.shutdownAccumulo(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - }); - } - public static AccumuloStoreManager getAccumuloStoreManager() throws StorageException { try { Configuration config = getAccumuloStorageConfiguration(); @@ -85,7 +41,7 @@ public static Configuration getAccumuloGraphConfiguration() { Configuration accumuloConfig = storageConfig.subset(AccumuloStoreManager.ACCUMULO_CONFIGURATION_NAMESPACE); - accumuloConfig.addProperty(AccumuloStoreManager.ACCUMULO_INTSANCE_KEY, "EtCloud"); + accumuloConfig.addProperty(AccumuloStoreManager.ACCUMULO_INTSANCE_KEY, "MockCloud"); accumuloConfig.addProperty(AccumuloStoreManager.ACCUMULO_USER_KEY, "root"); accumuloConfig.addProperty(AccumuloStoreManager.ACCUMULO_PASSWORD_KEY, ""); diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/blueprints/AccumuloBlueprintsTest.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/blueprints/AccumuloBlueprintsTest.java index 3c617bd858..9137fef98a 100644 --- a/titan-accumulo/src/test/java/com/thinkaurelius/titan/blueprints/AccumuloBlueprintsTest.java +++ b/titan-accumulo/src/test/java/com/thinkaurelius/titan/blueprints/AccumuloBlueprintsTest.java @@ -4,7 +4,6 @@ import com.thinkaurelius.titan.core.TitanFactory; import com.thinkaurelius.titan.diskstorage.StorageException; import com.thinkaurelius.titan.diskstorage.accumulo.AccumuloStoreManager; -import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration; import com.tinkerpop.blueprints.Graph; import java.io.IOException; diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/MockAccumuloStoreManager.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/MockAccumuloStoreManager.java index 96848849d4..81709f792a 100644 --- a/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/MockAccumuloStoreManager.java +++ b/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/MockAccumuloStoreManager.java @@ -10,20 +10,21 @@ import org.apache.commons.configuration.Configuration; /** - * + * Store manager that injects Mock Accumulo instance. + * * @author edeprit */ public class MockAccumuloStoreManager extends AccumuloStoreManager { - + static { - instanceInjector = new AccumuloInstanceInjector() { + instanceFactory = new AccumuloInstanceFactory() { @Override public Instance getInstance(String instanceName, String zooKeepers) { return new MockInstance(instanceName); - } + } }; } - + public MockAccumuloStoreManager(Configuration config) throws StorageException { super(config); } diff --git a/titan-accumulo/src/test/resources/log4j.properties b/titan-accumulo/src/test/resources/log4j.properties index 9e12d73134..1f8f2d4c63 100644 --- a/titan-accumulo/src/test/resources/log4j.properties +++ b/titan-accumulo/src/test/resources/log4j.properties @@ -9,6 +9,7 @@ log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n # Set root logger level to the designated level and its only appender to A1. log4j.rootLogger=INFO, A1 +log4j.logger.org.apache.accumulo=INFO log4j.logger.org.apache.cassandra=INFO log4j.logger.org.apache.hadoop=INFO log4j.logger.org.apache.zookeeper=INFO \ No newline at end of file From 1481288701b64d80a8c8bb53f77c08dcbf8ba0ce Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Tue, 17 Sep 2013 22:59:04 -0400 Subject: [PATCH 26/50] Refactor getSlice in AccumuloKeyColumnValueStore. --- .../accumulo/AccumuloKeyColumnValueStore.java | 33 +++++++++---------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java index 5d9ebb3948..b77fe89b71 100644 --- a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java +++ b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java @@ -21,7 +21,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.Map; import org.apache.accumulo.core.client.BatchScanner; @@ -142,20 +141,15 @@ public List> getSlice(List keys, SliceQuery query, Sto throw new PermanentStorageException(ex); } - List keysText = Lists.newArrayList(); - for (StaticBuffer key : keys) { - keysText.add(new Text(key.as(StaticBuffer.ARRAY_FACTORY))); - } - Collection ranges = Lists.newArrayList(); - for (Text key : keysText) { + for (StaticBuffer key : keys) { ranges.add(getRange(key, query)); } scanner.setRanges(ranges); - Map> entries = Maps.newHashMap(); - for (Text key : keysText) { + Map> entries = Maps.newHashMap(); + for (StaticBuffer key : keys) { entries.put(key, Lists.newArrayList()); } @@ -165,7 +159,7 @@ public List> getSlice(List keys, SliceQuery query, Sto break; } - Text key = kv.getKey().getRow(); + StaticBuffer key = new StaticArrayBuffer(kv.getKey().getRow().getBytes()); byte[] colQual = kv.getKey().getColumnQualifier().getBytes(); byte[] value = kv.getValue().get(); @@ -174,9 +168,11 @@ public List> getSlice(List keys, SliceQuery query, Sto entries.get(key).add(entry); count++; } + + scanner.close(); List> results = Lists.newArrayList(); - for (Text key : keysText) { + for (StaticBuffer key : keys) { results.add(entries.get(key)); } @@ -366,26 +362,27 @@ private Range getRange(StaticBuffer startKey, StaticBuffer endKey) { } private Range getRange(KeySliceQuery query) { - Text key = new Text(query.getKey().as(StaticBuffer.ARRAY_FACTORY)); - return getRange(key, query); + return getRange(query.getKey(), query); } - private Range getRange(Text key, SliceQuery query) { + private Range getRange(StaticBuffer key, SliceQuery query) { + Text row = new Text(key.as(StaticBuffer.ARRAY_FACTORY)); + Key startKey; Key endKey; if (query.getSliceStart().length() > 0) { - startKey = new Key(key, columnFamilyText, + startKey = new Key(row, columnFamilyText, new Text(query.getSliceStart().as(StaticBuffer.ARRAY_FACTORY))); } else { - startKey = new Key(key, columnFamilyText); + startKey = new Key(row, columnFamilyText); } if (query.getSliceEnd().length() > 0) { - endKey = new Key(key, columnFamilyText, + endKey = new Key(row, columnFamilyText, new Text(query.getSliceEnd().as(StaticBuffer.ARRAY_FACTORY))); } else { - endKey = new Key(key, columnFamilyText); + endKey = new Key(row, columnFamilyText); } return new Range(startKey, true, endKey, false); From 93e0e422fd6045316d3a90bafde5a3c660f9f970 Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Wed, 18 Sep 2013 13:28:09 -0400 Subject: [PATCH 27/50] Refactored getSlice in AccumuloKeyColumnValueStore to use executor service. --- .../accumulo/AccumuloKeyColumnValueStore.java | 107 ++++++++---------- .../accumulo/AccumuloStoreManager.java | 2 +- .../iterators}/ColumnRangeFilter.java | 22 +--- .../iterators}/ColumnRangeFilterTest.java | 2 +- 4 files changed, 54 insertions(+), 79 deletions(-) rename titan-accumulo/src/main/java/{org/apache/accumulo/core/iterators/user => com/thinkaurelius/titan/diskstorage/accumulo/iterators}/ColumnRangeFilter.java (82%) rename titan-accumulo/src/test/java/{org/apache/accumulo/core/iterators/user => com/thinkaurelius/titan/diskstorage/accumulo/iterators}/ColumnRangeFilterTest.java (98%) diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java index b77fe89b71..e90c7ab35c 100644 --- a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java +++ b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java @@ -1,9 +1,10 @@ package com.thinkaurelius.titan.diskstorage.accumulo; +import com.google.common.base.Function; import com.google.common.base.Preconditions; +import com.google.common.collect.Collections2; import com.google.common.collect.Iterators; import com.google.common.collect.Lists; -import com.google.common.collect.Maps; import com.google.common.collect.PeekingIterator; import com.thinkaurelius.titan.diskstorage.PermanentStorageException; import com.thinkaurelius.titan.diskstorage.StaticBuffer; @@ -19,11 +20,9 @@ import com.thinkaurelius.titan.diskstorage.util.RecordIterator; import com.thinkaurelius.titan.diskstorage.util.StaticArrayBuffer; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; -import org.apache.accumulo.core.client.BatchScanner; import org.apache.accumulo.core.client.BatchWriter; import org.apache.accumulo.core.client.ClientSideIteratorScanner; import org.slf4j.Logger; @@ -39,7 +38,14 @@ import org.apache.accumulo.core.data.Range; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.iterators.FirstEntryInRowIterator; -import org.apache.accumulo.core.iterators.user.ColumnRangeFilter; +import com.thinkaurelius.titan.diskstorage.accumulo.iterators.ColumnRangeFilter; +import java.util.Collection; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.logging.Level; import org.apache.accumulo.core.security.Authorizations; import org.apache.hadoop.io.Text; @@ -51,13 +57,13 @@ public class AccumuloKeyColumnValueStore implements KeyColumnValueStore { private static final Logger logger = LoggerFactory.getLogger(AccumuloKeyColumnValueStore.class); - // Default deleter, scanner, writer parameters + // Default parameters + private static final int NUM_THREADS_DEFAULT = Runtime.getRuntime().availableProcessors(); private static final Authorizations AUTHORIZATIONS_DEFAULT = new Authorizations(); // Instance variables private final Connector connector; // thread-safe private final String tableName; private final String columnFamily; - // This is columnFamily.getBytes() private final byte[] columnFamilyBytes; private final Text columnFamilyText; private final AccumuloBatchConfiguration batchConfiguration; @@ -116,67 +122,49 @@ public List getSlice(KeySliceQuery query, StoreTransaction txh) throws St } int count = 1; - List entries = new ArrayList(); - for (Map.Entry entry : scanner) { + List slice = new ArrayList(); + for (Map.Entry kv : scanner) { if (count > query.getLimit()) { break; } - - byte[] colQual = entry.getKey().getColumnQualifier().getBytes(); - byte[] value = entry.getValue().get(); - entries.add(StaticBufferEntry.of(new StaticArrayBuffer(colQual), new StaticArrayBuffer(value))); + slice.add(getEntry(kv)); count++; } - return entries; + return slice; } @Override - public List> getSlice(List keys, SliceQuery query, StoreTransaction txh) throws StorageException { - BatchScanner scanner; - try { - scanner = batchConfiguration.createBatchScanner(connector, tableName, AUTHORIZATIONS_DEFAULT); - } catch (TableNotFoundException ex) { - logger.error("Can't find Titan store " + tableName, ex); - throw new PermanentStorageException(ex); - } - - Collection ranges = Lists.newArrayList(); - for (StaticBuffer key : keys) { - ranges.add(getRange(key, query)); + public List> getSlice(List keys, final SliceQuery query, final StoreTransaction txh) throws StorageException { + List>> tasks = Lists.newArrayList(); + for (final StaticBuffer key : keys) { + tasks.add(new Callable>() { + @Override + public List call() throws Exception { + return getSlice(new KeySliceQuery(key, query), txh); + } + }); } - scanner.setRanges(ranges); - - Map> entries = Maps.newHashMap(); - for (StaticBuffer key : keys) { - entries.put(key, Lists.newArrayList()); - } + ExecutorService executor = Executors.newFixedThreadPool(NUM_THREADS_DEFAULT); + try { + List>> futures = executor.invokeAll(tasks); - int count = 1; - for (Map.Entry kv : scanner) { - if (count > query.getLimit()) { - break; + List> slices = Lists.newArrayList(); + for (Future> future : futures) { + slices.add(future.get()); } - StaticBuffer key = new StaticArrayBuffer(kv.getKey().getRow().getBytes()); - - byte[] colQual = kv.getKey().getColumnQualifier().getBytes(); - byte[] value = kv.getValue().get(); - Entry entry = StaticBufferEntry.of(new StaticArrayBuffer(colQual), new StaticArrayBuffer(value)); - - entries.get(key).add(entry); - count++; - } - - scanner.close(); - - List> results = Lists.newArrayList(); - for (StaticBuffer key : keys) { - results.add(entries.get(key)); + return slices; + } catch (InterruptedException ex) { + logger.error("Interrupted getSlice on Titan store " + tableName, ex); + throw new PermanentStorageException(ex); + } catch (ExecutionException ex) { + logger.error("Error executing getSlice on Titan store " + tableName, ex); + throw new PermanentStorageException(ex.getCause()); + } finally { + executor.shutdown(); } - - return results; } @Override @@ -346,6 +334,12 @@ private Scanner getKeySliceScanner(StaticBuffer startKey, StaticBuffer endKey, return scanner; } + private static Entry getEntry(Map.Entry keyValue) { + byte[] colQual = keyValue.getKey().getColumnQualifier().getBytes(); + byte[] value = keyValue.getValue().get(); + return StaticBufferEntry.of(new StaticArrayBuffer(colQual), new StaticArrayBuffer(value)); + } + private Range getRange(StaticBuffer startKey, StaticBuffer endKey) { Text startRow = null; Text endRow = null; @@ -367,7 +361,7 @@ private Range getRange(KeySliceQuery query) { private Range getRange(StaticBuffer key, SliceQuery query) { Text row = new Text(key.as(StaticBuffer.ARRAY_FACTORY)); - + Key startKey; Key endKey; @@ -426,12 +420,7 @@ public boolean hasNext() throws StorageException { public Entry next() throws StorageException { ensureOpen(); Map.Entry kv = currentRow.next(); - - byte[] colQual = kv.getKey().getColumnQualifier().getBytes(); - byte[] value = kv.getValue().get(); - Entry entry = StaticBufferEntry.of(new StaticArrayBuffer(colQual), new StaticArrayBuffer(value)); - - return entry; + return getEntry(kv); } @Override diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java index 69ba3ab0bc..4ae3e6464a 100644 --- a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java +++ b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java @@ -45,7 +45,7 @@ public class AccumuloStoreManager extends DistributedStoreManager implements KeyColumnValueStoreManager { private static final Logger logger = LoggerFactory.getLogger(AccumuloStoreManager.class); - // Default deleter, scanner, writer parameters + // Default parameters private static final Authorizations AUTHORIZATIONS_DEFAULT = new Authorizations(); // Configuration namespace public static final String ACCUMULO_CONFIGURATION_NAMESPACE = "accumulo-config"; diff --git a/titan-accumulo/src/main/java/org/apache/accumulo/core/iterators/user/ColumnRangeFilter.java b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/iterators/ColumnRangeFilter.java similarity index 82% rename from titan-accumulo/src/main/java/org/apache/accumulo/core/iterators/user/ColumnRangeFilter.java rename to titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/iterators/ColumnRangeFilter.java index b2f2aaaeca..61b142da5c 100644 --- a/titan-accumulo/src/main/java/org/apache/accumulo/core/iterators/user/ColumnRangeFilter.java +++ b/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/iterators/ColumnRangeFilter.java @@ -1,20 +1,4 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.core.iterators.user; +package com.thinkaurelius.titan.diskstorage.accumulo.iterators; import java.io.IOException; @@ -29,7 +13,9 @@ import org.apache.accumulo.core.iterators.SortedKeyValueIterator; /** - * A Filter that matches entries based on Java regular expressions. + * A Filter that matches entries based on range of column qualifiers. + * + * @author Etienne Deprit */ public class ColumnRangeFilter extends Filter { diff --git a/titan-accumulo/src/test/java/org/apache/accumulo/core/iterators/user/ColumnRangeFilterTest.java b/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/iterators/ColumnRangeFilterTest.java similarity index 98% rename from titan-accumulo/src/test/java/org/apache/accumulo/core/iterators/user/ColumnRangeFilterTest.java rename to titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/iterators/ColumnRangeFilterTest.java index cbcb0a6620..eb58738230 100644 --- a/titan-accumulo/src/test/java/org/apache/accumulo/core/iterators/user/ColumnRangeFilterTest.java +++ b/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/iterators/ColumnRangeFilterTest.java @@ -1,4 +1,4 @@ -package org.apache.accumulo.core.iterators.user; +package com.thinkaurelius.titan.diskstorage.accumulo.iterators; import java.io.IOException; import java.util.ArrayList; From bdad55031ebdd4aeeaf3b08b4f1ea3f0eeb675cc Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Fri, 20 Sep 2013 13:11:06 -0400 Subject: [PATCH 28/50] Code clean up and documentation. Passes all unit tests except concurrentReadWriteOnSingleTransaction. --- titan-accumulo/pom.xml | 142 +---------------- titan-accumulo/src/assembly/distribution.xml | 100 ------------ titan-accumulo/titan-accumulo-core/pom.xml | 149 ++++++++++++++++++ .../accumulo/AccumuloBatchConfiguration.java | 0 .../accumulo/AccumuloInstanceFactory.java | 5 +- .../accumulo/AccumuloKeyColumnValueStore.java | 72 +++------ .../accumulo/AccumuloStoreConfiguration.java | 39 ++++- .../accumulo/AccumuloStoreManager.java | 113 +++++++------ .../accumulo/AccumuloTransaction.java | 5 +- .../accumulo/util/CallableFunction.java | 18 +++ .../accumulo/util/ConcurrentLists.java | 113 +++++++++++++ .../src/main/resources/log4j.properties | 0 .../src/main/resources/titan.properties | 0 .../titan/AccumuloStorageSetup.java | 13 +- .../blueprints/AccumuloBlueprintsTest.java | 0 .../accumulo/AccumuloKeyColumnValueTest.java | 0 .../AccumuloLockKeyColumnValueStoreTest.java | 0 ...muloMultiWriteKeyColumnValueStoreTest.java | 0 .../accumulo/MockAccumuloStoreManager.java | 4 +- .../accumulo/AccumuloGraphConcurrentTest.java | 0 .../AccumuloGraphPerformanceTest.java | 0 .../graphdb/accumulo/AccumuloGraphTest.java | 0 .../src/test/resources/log4j.properties | 0 .../src/test/resources/rexster-fragment.xml | 0 .../titan-accumulo-iterators/pom.xml | 149 ++++++++++++++++++ .../accumulo/iterators/ColumnRangeFilter.java | 3 +- .../src/main/resources/log4j.properties | 14 ++ .../src/main/resources/titan.properties | 2 + .../iterators/ColumnRangeFilterTest.java | 0 .../iterators/DefaultIteratorEnvironment.java | 0 .../src/test/resources/log4j.properties | 15 ++ .../src/test/resources/rexster-fragment.xml | 13 ++ 32 files changed, 612 insertions(+), 357 deletions(-) delete mode 100644 titan-accumulo/src/assembly/distribution.xml create mode 100644 titan-accumulo/titan-accumulo-core/pom.xml rename titan-accumulo/{ => titan-accumulo-core}/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloBatchConfiguration.java (100%) rename titan-accumulo/{ => titan-accumulo-core}/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloInstanceFactory.java (91%) rename titan-accumulo/{ => titan-accumulo-core}/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java (87%) rename titan-accumulo/{ => titan-accumulo-core}/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreConfiguration.java (79%) rename titan-accumulo/{ => titan-accumulo-core}/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java (95%) rename titan-accumulo/{ => titan-accumulo-core}/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloTransaction.java (58%) create mode 100644 titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/util/CallableFunction.java create mode 100644 titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/util/ConcurrentLists.java rename titan-accumulo/{ => titan-accumulo-core}/src/main/resources/log4j.properties (100%) rename titan-accumulo/{ => titan-accumulo-core}/src/main/resources/titan.properties (100%) rename titan-accumulo/{ => titan-accumulo-core}/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java (86%) rename titan-accumulo/{ => titan-accumulo-core}/src/test/java/com/thinkaurelius/titan/blueprints/AccumuloBlueprintsTest.java (100%) rename titan-accumulo/{ => titan-accumulo-core}/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueTest.java (100%) rename titan-accumulo/{ => titan-accumulo-core}/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloLockKeyColumnValueStoreTest.java (100%) rename titan-accumulo/{ => titan-accumulo-core}/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloMultiWriteKeyColumnValueStoreTest.java (100%) rename titan-accumulo/{ => titan-accumulo-core}/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/MockAccumuloStoreManager.java (89%) rename titan-accumulo/{ => titan-accumulo-core}/src/test/java/com/thinkaurelius/titan/graphdb/accumulo/AccumuloGraphConcurrentTest.java (100%) rename titan-accumulo/{ => titan-accumulo-core}/src/test/java/com/thinkaurelius/titan/graphdb/accumulo/AccumuloGraphPerformanceTest.java (100%) rename titan-accumulo/{ => titan-accumulo-core}/src/test/java/com/thinkaurelius/titan/graphdb/accumulo/AccumuloGraphTest.java (100%) rename titan-accumulo/{ => titan-accumulo-core}/src/test/resources/log4j.properties (100%) rename titan-accumulo/{ => titan-accumulo-core}/src/test/resources/rexster-fragment.xml (100%) create mode 100644 titan-accumulo/titan-accumulo-iterators/pom.xml rename titan-accumulo/{ => titan-accumulo-iterators}/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/iterators/ColumnRangeFilter.java (99%) create mode 100644 titan-accumulo/titan-accumulo-iterators/src/main/resources/log4j.properties create mode 100644 titan-accumulo/titan-accumulo-iterators/src/main/resources/titan.properties rename titan-accumulo/{ => titan-accumulo-iterators}/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/iterators/ColumnRangeFilterTest.java (100%) rename titan-accumulo/{ => titan-accumulo-iterators}/src/test/java/org/apache/accumulo/core/iterators/DefaultIteratorEnvironment.java (100%) create mode 100644 titan-accumulo/titan-accumulo-iterators/src/test/resources/log4j.properties create mode 100644 titan-accumulo/titan-accumulo-iterators/src/test/resources/rexster-fragment.xml diff --git a/titan-accumulo/pom.xml b/titan-accumulo/pom.xml index be382af786..70f513c86a 100644 --- a/titan-accumulo/pom.xml +++ b/titan-accumulo/pom.xml @@ -8,141 +8,13 @@ ../pom.xml titan-accumulo + pom Titan-Accumulo: Distributed Graph Database http://thinkaurelius.github.com/titan/ - - - com.thinkaurelius.titan - titan-core - ${project.version} - - - com.thinkaurelius.titan - titan-test - ${project.version} - test - - - org.apache.accumulo - accumulo-core - 1.4.3 - - - org.apache.accumulo - accumulo-start - 1.4.3 - - - commons-logging-api - commons-logging - - - - - org.apache.hadoop - hadoop-core - 1.0.4 - - - - - comprehensive - - - - maven-surefire-plugin - 2.12.1 - - -Xms256m -Xmx3G - - **/*Test.java - - - **/Internal*.java - **/*Suite.java - - always - - - - - - - - ${basedir}/target - ${project.artifactId}-${project.version} - - - ${basedir}/src/main/resources - true - - - - - ${basedir}/src/test/resources - - - - - - maven-jar-plugin - - - only-iterator - - jar - - package - - only-iterator - - org/apache/accumulo/core/iterators/user/* - - - - - - - maven-dependency-plugin - 2.7 - - - test-compile - - copy-dependencies - - - target/test-lib - false - false - true - - - - - - maven-surefire-plugin - - -Xms256m -Xmx1G ${jacoco.opts} - - - **/*PerformanceTest.java - **/*ConcurrentTest.java - - - - - - maven-resources-plugin - - - org.jacoco - jacoco-maven-plugin - - - maven-antrun-plugin - - - - + + + titan-accumulo-core + titan-accumulo-iterators + + \ No newline at end of file diff --git a/titan-accumulo/src/assembly/distribution.xml b/titan-accumulo/src/assembly/distribution.xml deleted file mode 100644 index 3004a3caeb..0000000000 --- a/titan-accumulo/src/assembly/distribution.xml +++ /dev/null @@ -1,100 +0,0 @@ - - distribution - - zip - - - - - 0775 - ../titan-all/src/main/bin - bin - - *.sh - *.bat - - - - - 0775 - ./bin - bin - - *.sh - *.bat - - - - - ../bin/hbase.local - - - - ./config - config - - - src - - - ../doc - - - ../config - - - ../target/site/apidocs - doc/javadoc - - - target/*.jar - /lib - - - ../bin - bin - - hbase* - - - - - - pom.xml - src - - - ../LICENSE.txt - / - - - ../CHANGELOG.textile - / - - - ../UPGRADE.textile - / - - - ../NOTICE.txt - / - - - ../titan-all/src/main/bin/README.txt - / - - - - - - /lib - false - compile - - - /lib - false - provided - - - diff --git a/titan-accumulo/titan-accumulo-core/pom.xml b/titan-accumulo/titan-accumulo-core/pom.xml new file mode 100644 index 0000000000..8505607652 --- /dev/null +++ b/titan-accumulo/titan-accumulo-core/pom.xml @@ -0,0 +1,149 @@ + + 4.0.0 + + com.thinkaurelius.titan + titan-accumulo + 0.4.0-SNAPSHOT + ../pom.xml + + titan-accumulo-core + + + com.thinkaurelius.titan + titan-accumulo-iterators + ${project.version} + + + com.thinkaurelius.titan + titan-core + ${project.version} + + + com.thinkaurelius.titan + titan-test + ${project.version} + test + + + org.apache.accumulo + accumulo-core + 1.4.3 + + + libthrift + org.apache.thrift + + + + + org.apache.accumulo + accumulo-start + 1.4.3 + + + commons-logging-api + commons-logging + + + + + org.apache.zookeeper + zookeeper + 3.3.6 + + + org.apache.hadoop + hadoop-core + 1.0.4 + + + org.apache.thrift + libthrift + 0.6.1 + + + + + comprehensive + + + + maven-surefire-plugin + 2.12.1 + + -Xms256m -Xmx4G + + **/*Test.java + + + **/Internal*.java + **/*Suite.java + + always + + + + + + + + ${basedir}/target + ${project.artifactId}-${project.version} + + + ${basedir}/src/main/resources + true + + + + + ${basedir}/src/test/resources + + + + + + maven-dependency-plugin + 2.7 + + + test-compile + + copy-dependencies + + + target/test-lib + false + false + true + + + + + + maven-surefire-plugin + + -Xms256m -Xmx1G ${jacoco.opts} + + + **/*PerformanceTest.java + **/*ConcurrentTest.java + + + + + + maven-resources-plugin + + + org.jacoco + jacoco-maven-plugin + + + maven-antrun-plugin + + + + + \ No newline at end of file diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloBatchConfiguration.java b/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloBatchConfiguration.java similarity index 100% rename from titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloBatchConfiguration.java rename to titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloBatchConfiguration.java diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloInstanceFactory.java b/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloInstanceFactory.java similarity index 91% rename from titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloInstanceFactory.java rename to titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloInstanceFactory.java index 71cba81a3a..7c28eb4ff1 100644 --- a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloInstanceFactory.java +++ b/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloInstanceFactory.java @@ -11,7 +11,10 @@ public interface AccumuloInstanceFactory { public Instance getInstance(String instanceName, String zooKeepers); - // + + /* + * Default Zookeeper instance factory. + */ public static final AccumuloInstanceFactory ZOOKEEPER_INSTANCE_FACTORY = new AccumuloInstanceFactory() { @Override diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java b/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java similarity index 87% rename from titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java rename to titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java index e90c7ab35c..93295548a5 100644 --- a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java +++ b/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java @@ -1,10 +1,7 @@ package com.thinkaurelius.titan.diskstorage.accumulo; -import com.google.common.base.Function; import com.google.common.base.Preconditions; -import com.google.common.collect.Collections2; import com.google.common.collect.Iterators; -import com.google.common.collect.Lists; import com.google.common.collect.PeekingIterator; import com.thinkaurelius.titan.diskstorage.PermanentStorageException; import com.thinkaurelius.titan.diskstorage.StaticBuffer; @@ -39,18 +36,13 @@ import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.iterators.FirstEntryInRowIterator; import com.thinkaurelius.titan.diskstorage.accumulo.iterators.ColumnRangeFilter; -import java.util.Collection; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.logging.Level; +import com.thinkaurelius.titan.diskstorage.accumulo.util.CallableFunction; +import com.thinkaurelius.titan.diskstorage.accumulo.util.ConcurrentLists; import org.apache.accumulo.core.security.Authorizations; import org.apache.hadoop.io.Text; /** - * Key-Column value store for Accumulo + * Key-Column value store for Accumulo. * * @author Etienne Deprit */ @@ -67,17 +59,17 @@ public class AccumuloKeyColumnValueStore implements KeyColumnValueStore { private final byte[] columnFamilyBytes; private final Text columnFamilyText; private final AccumuloBatchConfiguration batchConfiguration; - private final boolean clientSideIterators; + private final boolean serverSideIterators; AccumuloKeyColumnValueStore(Connector connector, String tableName, String columnFamily, - AccumuloBatchConfiguration batchConfiguration, boolean clientSideIterators) { + AccumuloBatchConfiguration batchConfiguration, boolean serverSideIterators) { this.connector = connector; this.tableName = tableName; this.columnFamily = columnFamily; this.columnFamilyBytes = columnFamily.getBytes(); this.columnFamilyText = new Text(columnFamily); this.batchConfiguration = batchConfiguration; - this.clientSideIterators = clientSideIterators; + this.serverSideIterators = serverSideIterators; } @Override @@ -136,35 +128,15 @@ public List getSlice(KeySliceQuery query, StoreTransaction txh) throws St @Override public List> getSlice(List keys, final SliceQuery query, final StoreTransaction txh) throws StorageException { - List>> tasks = Lists.newArrayList(); - for (final StaticBuffer key : keys) { - tasks.add(new Callable>() { - @Override - public List call() throws Exception { - return getSlice(new KeySliceQuery(key, query), txh); - } - }); - } - - ExecutorService executor = Executors.newFixedThreadPool(NUM_THREADS_DEFAULT); - try { - List>> futures = executor.invokeAll(tasks); - - List> slices = Lists.newArrayList(); - for (Future> future : futures) { - slices.add(future.get()); + List> slices = ConcurrentLists.transform(keys, + new CallableFunction>() { + @Override + public List apply(StaticBuffer key) throws Exception { + return getSlice(new KeySliceQuery(key, query), txh); } + }); - return slices; - } catch (InterruptedException ex) { - logger.error("Interrupted getSlice on Titan store " + tableName, ex); - throw new PermanentStorageException(ex); - } catch (ExecutionException ex) { - logger.error("Error executing getSlice on Titan store " + tableName, ex); - throw new PermanentStorageException(ex.getCause()); - } finally { - executor.shutdown(); - } + return slices; } @Override @@ -220,11 +192,11 @@ private static List makeBatch(Text colFamily, Text key, List ad } /** - * Convert deletions to a Delete mutation. + * Convert Titan deletions into Accumulo delete {@code Mutation}. * - * @param colFamily The name of the ColumnFamily deletions belong to - * @param key The row key - * @param deletions The name of the columns to delete (a.k.a deletions) + * @param colFamily Name of column family for deletions + * @param key Row key + * @param deletions Name of column qualifiers to delete * * @return Delete command or null if deletions were null or empty. */ @@ -240,11 +212,11 @@ private static Mutation makeDeleteMutation(Text colFamily, Text key, ListAccumuloStoreConfiguration is thread-safe. * * @author Etienne Deprit @@ -44,10 +45,26 @@ public class AccumuloStoreConfiguration { private final Text rowIdText; private final Text colFamilyText; + /** + * Construct Accumulo store configuration that gets and sets properties. + * + * @param connector Connection to Accumulo instance + * @param tableName Accumulo table backing Titan store + * @param batchConfiguration Configuration for batch operations + */ public AccumuloStoreConfiguration(Connector connector, String tableName, AccumuloBatchConfiguration batchConfiguration) { this(connector, tableName, batchConfiguration, ROW_ID_DEFAULT, COL_FAMILY_DEFAULT); } + /** + * Construct Accumulo store configuration that gets and sets properties. + * + * @param connector Connection to Accumulo instance + * @param tableName Accumulo table backing Titan store + * @param batchConfiguration Configuration for batch operations + * @param rowId Row containing properties. + * @param colFamily Column family for key-value pairs. + */ public AccumuloStoreConfiguration(Connector connector, String tableName, AccumuloBatchConfiguration batchConfiguration, String rowId, String colFamily) { this.connector = connector; @@ -57,6 +74,15 @@ public AccumuloStoreConfiguration(Connector connector, String tableName, this.colFamilyText = new Text(colFamily); } + /** + * Get Accumulo store property for { + * + * @ key}. + * + * @param key Property to get + * @return Property value + * @throws StorageException + */ public String getConfigurationProperty(String key) throws StorageException { Preconditions.checkArgument(key != null, "Key cannot be null"); @@ -80,6 +106,13 @@ public String getConfigurationProperty(String key) throws StorageException { } } + /** + * Sets Accumulo store property for {@ key} to {@ value}. + * + * @param key Property to set + * @param value Property value + * @throws StorageException + */ public void setConfigurationProperty(String key, String value) throws StorageException { try { BatchWriter writer = batchConfiguration.createBatchWriter(connector, tableName); diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java b/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java similarity index 95% rename from titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java rename to titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java index 4ae3e6464a..88475d506c 100644 --- a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java +++ b/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java @@ -38,7 +38,7 @@ import org.slf4j.LoggerFactory; /** - * Storage Manager for Accumulo + * Storage Manager for Accumulo. * * @author Etienne Deprit */ @@ -54,11 +54,11 @@ public class AccumuloStoreManager extends DistributedStoreManager implements Key public static final String ACCUMULO_USER_KEY = "username"; public static final String ACCUMULO_PASSWORD_KEY = "password"; public static final String TABLE_NAME_KEY = "tablename"; - public static final String CLIENT_SIDE_ITERATORS_KEY = "client-side-iterators"; + public static final String SERVER_SIDE_ITERATORS_KEY = "server-side-iterators"; // Configuration defaults public static final String TABLE_NAME_DEFAULT = "titan"; public static final int PORT_DEFAULT = 9160; - public static final boolean CLIENT_SIDE_ITERATORS_DEFAULT = true; + public static final boolean SERVER_SIDE_ITERATORS_DEFAULT = false; // Instance injector public static AccumuloInstanceFactory instanceFactory = AccumuloInstanceFactory.ZOOKEEPER_INSTANCE_FACTORY; // Instance variables @@ -67,7 +67,7 @@ public class AccumuloStoreManager extends DistributedStoreManager implements Key private final String zooKeepers; private final String username; private final String password; - private final boolean clientSideIterators; + private final boolean serverSideIterators; private final Instance instance; // thread-safe private final Connector connector; // thread-safe private final ConcurrentMap openStores; @@ -89,7 +89,7 @@ public AccumuloStoreManager(Configuration config) throws StorageException { username = accumuloConfig.getString(ACCUMULO_USER_KEY); password = accumuloConfig.getString(ACCUMULO_PASSWORD_KEY); - clientSideIterators = accumuloConfig.getBoolean(CLIENT_SIDE_ITERATORS_KEY, CLIENT_SIDE_ITERATORS_DEFAULT); + serverSideIterators = accumuloConfig.getBoolean(SERVER_SIDE_ITERATORS_KEY, SERVER_SIDE_ITERATORS_DEFAULT); instance = instanceFactory.getInstance(instanceName, zooKeepers); @@ -116,7 +116,7 @@ public AccumuloStoreManager(Configuration config) throws StorageException { features.hasLocalKeyPartition = false; batchConfiguration = new AccumuloBatchConfiguration(); - + storeConfiguration = new AccumuloStoreConfiguration(connector, tableName, batchConfiguration); } @@ -146,7 +146,7 @@ public KeyColumnValueStore openDatabase(String dbName) throws StorageException { if (store == null) { AccumuloKeyColumnValueStore newStore = new AccumuloKeyColumnValueStore(connector, tableName, dbName, - batchConfiguration, clientSideIterators); + batchConfiguration, serverSideIterators); store = openStores.putIfAbsent(dbName, newStore); // atomic so only one store dbName @@ -155,10 +155,53 @@ public KeyColumnValueStore openDatabase(String dbName) throws StorageException { store = newStore; } } - + return store; } + @Override + public void clearStorage() throws StorageException { + TableOperations operations = connector.tableOperations(); + + // Check if table exists, if not we are done + if (!operations.exists(tableName)) { + logger.warn("clearStorage() called before table {} created, skipping.", tableName); + return; + } + + try { + BatchDeleter deleter = batchConfiguration.createBatchDeleter(connector, tableName, AUTHORIZATIONS_DEFAULT); + deleter.setRanges(Collections.singletonList(new Range())); + try { + deleter.delete(); + } catch (MutationsRejectedException ex) { + logger.error("Can't write mutations to " + tableName, ex); + throw new PermanentStorageException(ex); + } finally { + deleter.close(); + } + + } catch (TableNotFoundException ex) { + logger.error("Can't find Titan table " + tableName, ex); + throw new PermanentStorageException(ex); + } + } + + @Override + public String getConfigurationProperty(String key) throws StorageException { + return storeConfiguration.getConfigurationProperty(key); + } + + @Override + public void setConfigurationProperty(String key, String value) throws StorageException { + storeConfiguration.setConfigurationProperty(key, value); + } + + @Override + public StoreTransaction beginTransaction(ConsistencyLevel level) throws StorageException { + return new AccumuloTransaction(level); + } + @Override public void mutateMany(Map> mutations, StoreTransaction txh) throws StorageException { final long delTS = System.currentTimeMillis(); @@ -191,8 +234,10 @@ public void mutateMany(Map> mutations, St } /** - * Convert Titan internal Mutation representation into Accumulo native - * mutations. + * Convert Titan internal { + * + * @ KCVMutation} representation into Accumulo native { + * @ Mutation}. * * @param mutations Mutations to convert into Accumulo actions. * @param putTimestamp The timestamp to use for put mutations. @@ -258,54 +303,6 @@ private void ensureTableExists(String tableName) throws StorageException { } } - @Override - public StoreTransaction beginTransaction(ConsistencyLevel level) throws StorageException { - return new AccumuloTransaction(level); - } - - /** - * Delete all key/value pairs in the specified table. - * - * ATTENTION: Invoking this method causes data loss. - */ - @Override - public void clearStorage() throws StorageException { - TableOperations operations = connector.tableOperations(); - - // Check if table exists, if not we are done - if (!operations.exists(tableName)) { - logger.warn("clearStorage() called before table {} created, skipping.", tableName); - return; - } - - try { - BatchDeleter deleter = batchConfiguration.createBatchDeleter(connector, tableName, AUTHORIZATIONS_DEFAULT); - deleter.setRanges(Collections.singletonList(new Range())); - try { - deleter.delete(); - } catch (MutationsRejectedException ex) { - logger.error("Can't write mutations to " + tableName, ex); - throw new PermanentStorageException(ex); - } finally { - deleter.close(); - } - - } catch (TableNotFoundException ex) { - logger.error("Can't find Titan table " + tableName, ex); - throw new PermanentStorageException(ex); - } - } - - @Override - public String getConfigurationProperty(String key) throws StorageException { - return storeConfiguration.getConfigurationProperty(key); - } - - @Override - public void setConfigurationProperty(String key, String value) throws StorageException { - storeConfiguration.setConfigurationProperty(key, value); - } - private static void waitUntil(long until) { long now = System.currentTimeMillis(); diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloTransaction.java b/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloTransaction.java similarity index 58% rename from titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloTransaction.java rename to titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloTransaction.java index a52ca58fe4..1f3c24e53f 100644 --- a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloTransaction.java +++ b/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloTransaction.java @@ -4,10 +4,7 @@ import com.thinkaurelius.titan.diskstorage.keycolumnvalue.ConsistencyLevel; /** - * This class overrides and adds nothing compared with - * {@link com.thinkaurelius.titan.diskstorage.locking.consistentkey.ConsistentKeyLockTransaction}; however, it creates a transaction type specific - * to HBase, which lets us check for user errors like passing a Cassandra - * transaction into a HBase method. + * This creates a transaction type specific to Accumulo. * * @author Etienne Deprit */ diff --git a/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/util/CallableFunction.java b/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/util/CallableFunction.java new file mode 100644 index 0000000000..44c76478b6 --- /dev/null +++ b/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/util/CallableFunction.java @@ -0,0 +1,18 @@ +package com.thinkaurelius.titan.diskstorage.accumulo.util; + +/** + * Callable function that throws exceptions for use with {@code ConcurrentLists}. + * + * @author Etienne Deprit + */ +public interface CallableFunction { + + /** + * Application of this function to {@code input}. + * + * @param input Function parameter + * @return Computed value + * @throws Exception + */ + public T apply(F input) throws Exception; +} diff --git a/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/util/ConcurrentLists.java b/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/util/ConcurrentLists.java new file mode 100644 index 0000000000..97f8973a95 --- /dev/null +++ b/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/util/ConcurrentLists.java @@ -0,0 +1,113 @@ +package com.thinkaurelius.titan.diskstorage.accumulo.util; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +/** + * Concurrent transformations on lists. + * + * @author Etienne Deprit + */ +public class ConcurrentLists { + + /** + * Default size of executor thread pool. + */ + public static final int NUM_THREADS_DEFAULT = 4; + + /** + * Returns list that is concurrent application of {@code function} to each + * element of {@code fromList}. + * + * @param Element type of from list + * @param Element type of result list + * @param fromList Source list + * @param function Transformation function + * @return List of function applications + */ + public static List transform(List fromList, + final CallableFunction function) { + + return transform(fromList, function, NUM_THREADS_DEFAULT); + } + + /** + * Returns list that is concurrent application of {@code function} to each + * element of {@code fromList}. + * + * @param Element type of from list + * @param Element type of result list + * @param fromList Source list + * @param function Transformation function + * @param numThreads Size of thread pool + * @return List of function applications + */ + public static List transform(List fromList, + CallableFunction function, int numThreads) { + Preconditions.checkArgument(numThreads > 0, "numThreads must be > 0"); + + ExecutorService executor = null; + try { + executor = Executors.newFixedThreadPool(numThreads); + return transform(fromList, function, executor); + } finally { + if (executor != null) { + executor.shutdown(); + } + } + } + + /** + * Returns list that is concurrent application of {@code function} to each + * element of {@code fromList}. + * + * @param Element type of from list + * @param Element type of result list + * @param fromList Source list + * @param function Transformation function + * @param executor Executor service for threads + * @return List of function applications + */ + public static List transform(List fromList, + final CallableFunction function, ExecutorService executor) { + + List> tasks = Lists.newArrayListWithCapacity(fromList.size()); + for (final F from : fromList) { + tasks.add( + new Callable() { + @Override + public T call() throws Exception { + return function.apply(from); + } + }); + } + + List results = Lists.newArrayListWithCapacity(fromList.size()); + try { + List> futures = executor.invokeAll(tasks); + + for (Future future : futures) { + results.add(future.get()); + } + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + } catch (ExecutionException ex) { + Throwable t = ex.getCause(); + if (t instanceof RuntimeException) { + throw (RuntimeException) t; + } else if (t instanceof Error) { + throw (Error) t; + } else { + throw new IllegalStateException(t); + } + } + + return results; + } +} \ No newline at end of file diff --git a/titan-accumulo/src/main/resources/log4j.properties b/titan-accumulo/titan-accumulo-core/src/main/resources/log4j.properties similarity index 100% rename from titan-accumulo/src/main/resources/log4j.properties rename to titan-accumulo/titan-accumulo-core/src/main/resources/log4j.properties diff --git a/titan-accumulo/src/main/resources/titan.properties b/titan-accumulo/titan-accumulo-core/src/main/resources/titan.properties similarity index 100% rename from titan-accumulo/src/main/resources/titan.properties rename to titan-accumulo/titan-accumulo-core/src/main/resources/titan.properties diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java b/titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java similarity index 86% rename from titan-accumulo/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java rename to titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java index b80e6e216f..17939bfc8a 100644 --- a/titan-accumulo/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java +++ b/titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java @@ -9,6 +9,11 @@ import java.io.IOException; import java.lang.reflect.Constructor; +/** + * Set up Accumulo storage back-end for unit tests. + * + * @author Etienne Deprit + */ public class AccumuloStorageSetup { public static AccumuloStoreManager getAccumuloStoreManager() throws StorageException { @@ -36,15 +41,17 @@ public static Configuration getAccumuloGraphConfiguration() { Configuration storageConfig = config.subset(GraphDatabaseConfiguration.STORAGE_NAMESPACE); storageConfig.addProperty(GraphDatabaseConfiguration.STORAGE_BACKEND_KEY, - "com.thinkaurelius.titan.diskstorage.accumulo.MockAccumuloStoreManager"); + "com.thinkaurelius.titan.diskstorage.accumulo.AccumuloStoreManager"); storageConfig.addProperty(GraphDatabaseConfiguration.HOSTNAME_KEY, "localhost"); Configuration accumuloConfig = storageConfig.subset(AccumuloStoreManager.ACCUMULO_CONFIGURATION_NAMESPACE); - accumuloConfig.addProperty(AccumuloStoreManager.ACCUMULO_INTSANCE_KEY, "MockCloud"); + accumuloConfig.addProperty(AccumuloStoreManager.ACCUMULO_INTSANCE_KEY, "devdb"); accumuloConfig.addProperty(AccumuloStoreManager.ACCUMULO_USER_KEY, "root"); - accumuloConfig.addProperty(AccumuloStoreManager.ACCUMULO_PASSWORD_KEY, ""); + accumuloConfig.addProperty(AccumuloStoreManager.ACCUMULO_PASSWORD_KEY, "toor"); + + accumuloConfig.addProperty(AccumuloStoreManager.SERVER_SIDE_ITERATORS_KEY, true); return config; } diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/blueprints/AccumuloBlueprintsTest.java b/titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/blueprints/AccumuloBlueprintsTest.java similarity index 100% rename from titan-accumulo/src/test/java/com/thinkaurelius/titan/blueprints/AccumuloBlueprintsTest.java rename to titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/blueprints/AccumuloBlueprintsTest.java diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueTest.java b/titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueTest.java similarity index 100% rename from titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueTest.java rename to titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueTest.java diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloLockKeyColumnValueStoreTest.java b/titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloLockKeyColumnValueStoreTest.java similarity index 100% rename from titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloLockKeyColumnValueStoreTest.java rename to titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloLockKeyColumnValueStoreTest.java diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloMultiWriteKeyColumnValueStoreTest.java b/titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloMultiWriteKeyColumnValueStoreTest.java similarity index 100% rename from titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloMultiWriteKeyColumnValueStoreTest.java rename to titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloMultiWriteKeyColumnValueStoreTest.java diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/MockAccumuloStoreManager.java b/titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/MockAccumuloStoreManager.java similarity index 89% rename from titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/MockAccumuloStoreManager.java rename to titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/MockAccumuloStoreManager.java index 81709f792a..bd8267dc81 100644 --- a/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/MockAccumuloStoreManager.java +++ b/titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/MockAccumuloStoreManager.java @@ -10,9 +10,9 @@ import org.apache.commons.configuration.Configuration; /** - * Store manager that injects Mock Accumulo instance. + * Store manager that injects mock Accumulo instance. * - * @author edeprit + * @author Etienne Deprit */ public class MockAccumuloStoreManager extends AccumuloStoreManager { diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/accumulo/AccumuloGraphConcurrentTest.java b/titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/graphdb/accumulo/AccumuloGraphConcurrentTest.java similarity index 100% rename from titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/accumulo/AccumuloGraphConcurrentTest.java rename to titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/graphdb/accumulo/AccumuloGraphConcurrentTest.java diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/accumulo/AccumuloGraphPerformanceTest.java b/titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/graphdb/accumulo/AccumuloGraphPerformanceTest.java similarity index 100% rename from titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/accumulo/AccumuloGraphPerformanceTest.java rename to titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/graphdb/accumulo/AccumuloGraphPerformanceTest.java diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/accumulo/AccumuloGraphTest.java b/titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/graphdb/accumulo/AccumuloGraphTest.java similarity index 100% rename from titan-accumulo/src/test/java/com/thinkaurelius/titan/graphdb/accumulo/AccumuloGraphTest.java rename to titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/graphdb/accumulo/AccumuloGraphTest.java diff --git a/titan-accumulo/src/test/resources/log4j.properties b/titan-accumulo/titan-accumulo-core/src/test/resources/log4j.properties similarity index 100% rename from titan-accumulo/src/test/resources/log4j.properties rename to titan-accumulo/titan-accumulo-core/src/test/resources/log4j.properties diff --git a/titan-accumulo/src/test/resources/rexster-fragment.xml b/titan-accumulo/titan-accumulo-core/src/test/resources/rexster-fragment.xml similarity index 100% rename from titan-accumulo/src/test/resources/rexster-fragment.xml rename to titan-accumulo/titan-accumulo-core/src/test/resources/rexster-fragment.xml diff --git a/titan-accumulo/titan-accumulo-iterators/pom.xml b/titan-accumulo/titan-accumulo-iterators/pom.xml new file mode 100644 index 0000000000..b2e896c4ef --- /dev/null +++ b/titan-accumulo/titan-accumulo-iterators/pom.xml @@ -0,0 +1,149 @@ + + 4.0.0 + + com.thinkaurelius.titan + titan-accumulo + 0.4.0-SNAPSHOT + ../pom.xml + + titan-accumulo-iterators + + + org.apache.accumulo + accumulo-core + 1.4.3 + provided + + + libthrift + org.apache.thrift + + + + + org.apache.accumulo + accumulo-start + 1.4.3 + provided + + + commons-logging-api + commons-logging + + + + + org.apache.hadoop + hadoop-core + 1.0.4 + provided + + + junit + junit + 4.11 + provided + + + + + comprehensive + + + + maven-surefire-plugin + 2.12.1 + + -Xms256m -Xmx4G + + **/*Test.java + + + **/Internal*.java + **/*Suite.java + + always + + + + + + + + ${basedir}/target + ${project.artifactId}-${project.version} + + + ${basedir}/src/main/resources + true + + + + + ${basedir}/src/test/resources + + + + + + org.apache.maven.plugins + maven-shade-plugin + 2.0 + + + package + + shade + + + + + false + true + + + + maven-dependency-plugin + 2.7 + + + test-compile + + copy-dependencies + + + target/test-lib + false + false + true + + + + + + maven-surefire-plugin + + -Xms256m -Xmx1G ${jacoco.opts} + + + **/*PerformanceTest.java + **/*ConcurrentTest.java + + + + + + maven-resources-plugin + + + org.jacoco + jacoco-maven-plugin + + + maven-antrun-plugin + + + + + \ No newline at end of file diff --git a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/iterators/ColumnRangeFilter.java b/titan-accumulo/titan-accumulo-iterators/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/iterators/ColumnRangeFilter.java similarity index 99% rename from titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/iterators/ColumnRangeFilter.java rename to titan-accumulo/titan-accumulo-iterators/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/iterators/ColumnRangeFilter.java index 61b142da5c..fa8c558dc7 100644 --- a/titan-accumulo/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/iterators/ColumnRangeFilter.java +++ b/titan-accumulo/titan-accumulo-iterators/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/iterators/ColumnRangeFilter.java @@ -13,7 +13,8 @@ import org.apache.accumulo.core.iterators.SortedKeyValueIterator; /** - * A Filter that matches entries based on range of column qualifiers. + * A Filter that matches entries based on range of column qualifiers, + * modeled after {@code RegExFilter}. * * @author Etienne Deprit */ diff --git a/titan-accumulo/titan-accumulo-iterators/src/main/resources/log4j.properties b/titan-accumulo/titan-accumulo-iterators/src/main/resources/log4j.properties new file mode 100644 index 0000000000..a02cd1ff42 --- /dev/null +++ b/titan-accumulo/titan-accumulo-iterators/src/main/resources/log4j.properties @@ -0,0 +1,14 @@ +# A1 is set to be a ConsoleAppender. +log4j.appender.A1=org.apache.log4j.ConsoleAppender + +# A1 uses PatternLayout. +log4j.appender.A1.layout=org.apache.log4j.PatternLayout +log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n + +# Set root logger level to the designated level and its only appender to A1. +log4j.rootLogger=INFO, A1 + +log4j.logger.org.apache.accumulo=INFO +log4j.logger.org.apache.cassandra=INFO +log4j.logger.org.apache.hadoop=INFO +log4j.logger.org.apache.zookeeper=INFO diff --git a/titan-accumulo/titan-accumulo-iterators/src/main/resources/titan.properties b/titan-accumulo/titan-accumulo-iterators/src/main/resources/titan.properties new file mode 100644 index 0000000000..02dc2c4ce5 --- /dev/null +++ b/titan-accumulo/titan-accumulo-iterators/src/main/resources/titan.properties @@ -0,0 +1,2 @@ +titan.version=${project.version} +titan.compatible-versions=${titan.compatible.versions} \ No newline at end of file diff --git a/titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/iterators/ColumnRangeFilterTest.java b/titan-accumulo/titan-accumulo-iterators/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/iterators/ColumnRangeFilterTest.java similarity index 100% rename from titan-accumulo/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/iterators/ColumnRangeFilterTest.java rename to titan-accumulo/titan-accumulo-iterators/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/iterators/ColumnRangeFilterTest.java diff --git a/titan-accumulo/src/test/java/org/apache/accumulo/core/iterators/DefaultIteratorEnvironment.java b/titan-accumulo/titan-accumulo-iterators/src/test/java/org/apache/accumulo/core/iterators/DefaultIteratorEnvironment.java similarity index 100% rename from titan-accumulo/src/test/java/org/apache/accumulo/core/iterators/DefaultIteratorEnvironment.java rename to titan-accumulo/titan-accumulo-iterators/src/test/java/org/apache/accumulo/core/iterators/DefaultIteratorEnvironment.java diff --git a/titan-accumulo/titan-accumulo-iterators/src/test/resources/log4j.properties b/titan-accumulo/titan-accumulo-iterators/src/test/resources/log4j.properties new file mode 100644 index 0000000000..1f8f2d4c63 --- /dev/null +++ b/titan-accumulo/titan-accumulo-iterators/src/test/resources/log4j.properties @@ -0,0 +1,15 @@ +# A1 is set to be a FileAppender. +log4j.appender.A1=org.apache.log4j.FileAppender +log4j.appender.A1.File=target/test.log + +# A1 uses PatternLayout. +log4j.appender.A1.layout=org.apache.log4j.PatternLayout +log4j.appender.A1.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n + +# Set root logger level to the designated level and its only appender to A1. +log4j.rootLogger=INFO, A1 + +log4j.logger.org.apache.accumulo=INFO +log4j.logger.org.apache.cassandra=INFO +log4j.logger.org.apache.hadoop=INFO +log4j.logger.org.apache.zookeeper=INFO \ No newline at end of file diff --git a/titan-accumulo/titan-accumulo-iterators/src/test/resources/rexster-fragment.xml b/titan-accumulo/titan-accumulo-iterators/src/test/resources/rexster-fragment.xml new file mode 100644 index 0000000000..19f88f9236 --- /dev/null +++ b/titan-accumulo/titan-accumulo-iterators/src/test/resources/rexster-fragment.xml @@ -0,0 +1,13 @@ + + + false + home + + local + + + + tp:gremlin + + + From c4df9655fd9c1a76e640220a28d6fc8d042dfbd8 Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Fri, 20 Sep 2013 13:21:17 -0400 Subject: [PATCH 29/50] Restore mock Accumulo instance for unit tests in AccumuloStorageConfiguration. --- .../java/com/thinkaurelius/titan/AccumuloStorageSetup.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java b/titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java index 17939bfc8a..babaa8c7a9 100644 --- a/titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java +++ b/titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java @@ -41,7 +41,7 @@ public static Configuration getAccumuloGraphConfiguration() { Configuration storageConfig = config.subset(GraphDatabaseConfiguration.STORAGE_NAMESPACE); storageConfig.addProperty(GraphDatabaseConfiguration.STORAGE_BACKEND_KEY, - "com.thinkaurelius.titan.diskstorage.accumulo.AccumuloStoreManager"); + "com.thinkaurelius.titan.diskstorage.accumulo.MockAccumuloStoreManager"); storageConfig.addProperty(GraphDatabaseConfiguration.HOSTNAME_KEY, "localhost"); Configuration accumuloConfig = storageConfig.subset(AccumuloStoreManager.ACCUMULO_CONFIGURATION_NAMESPACE); @@ -49,9 +49,9 @@ public static Configuration getAccumuloGraphConfiguration() { accumuloConfig.addProperty(AccumuloStoreManager.ACCUMULO_INTSANCE_KEY, "devdb"); accumuloConfig.addProperty(AccumuloStoreManager.ACCUMULO_USER_KEY, "root"); - accumuloConfig.addProperty(AccumuloStoreManager.ACCUMULO_PASSWORD_KEY, "toor"); + accumuloConfig.addProperty(AccumuloStoreManager.ACCUMULO_PASSWORD_KEY, ""); - accumuloConfig.addProperty(AccumuloStoreManager.SERVER_SIDE_ITERATORS_KEY, true); + accumuloConfig.addProperty(AccumuloStoreManager.SERVER_SIDE_ITERATORS_KEY, false); return config; } From 3ffc740f8c28bee0c01f5873f0babdadbc7398ac Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Sun, 22 Sep 2013 19:19:30 -0400 Subject: [PATCH 30/50] Added titan-accumulo to titan-dist. --- titan-accumulo/pom.xml | 5 +- titan-accumulo/titan-accumulo-core/pom.xml | 1 + .../accumulo/AccumuloKeyColumnValueStore.java | 3 +- .../accumulo/AccumuloStoreManager.java | 7 +- .../titan-accumulo-iterators/pom.xml | 1 + titan-all/pom.xml | 8 ++- titan-dist/pom.xml | 1 + .../titan-dist-accumulo/assembly-accumulo.xml | 68 +++++++++++++++++++ titan-dist/titan-dist-accumulo/pom.xml | 67 ++++++++++++++++++ titan-dist/titan-dist-all/pom.xml | 11 +++ 10 files changed, 164 insertions(+), 8 deletions(-) create mode 100644 titan-dist/titan-dist-accumulo/assembly-accumulo.xml create mode 100644 titan-dist/titan-dist-accumulo/pom.xml diff --git a/titan-accumulo/pom.xml b/titan-accumulo/pom.xml index 70f513c86a..357420d026 100644 --- a/titan-accumulo/pom.xml +++ b/titan-accumulo/pom.xml @@ -13,8 +13,7 @@ http://thinkaurelius.github.com/titan/ - titan-accumulo-core titan-accumulo-iterators - - + titan-accumulo-core + \ No newline at end of file diff --git a/titan-accumulo/titan-accumulo-core/pom.xml b/titan-accumulo/titan-accumulo-core/pom.xml index 8505607652..5caf5a2ca8 100644 --- a/titan-accumulo/titan-accumulo-core/pom.xml +++ b/titan-accumulo/titan-accumulo-core/pom.xml @@ -8,6 +8,7 @@ ../pom.xml titan-accumulo-core + Titan-Accumulo: Graph Database Core com.thinkaurelius.titan diff --git a/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java b/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java index 93295548a5..b25ba7a1dc 100644 --- a/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java +++ b/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java @@ -6,6 +6,7 @@ import com.thinkaurelius.titan.diskstorage.PermanentStorageException; import com.thinkaurelius.titan.diskstorage.StaticBuffer; import com.thinkaurelius.titan.diskstorage.StorageException; +import com.thinkaurelius.titan.diskstorage.TemporaryStorageException; import com.thinkaurelius.titan.diskstorage.keycolumnvalue.Entry; import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyColumnValueStore; import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyIterator; @@ -157,7 +158,7 @@ public void mutate(StaticBuffer key, List additions, List d writer.flush(); } catch (MutationsRejectedException ex) { logger.error("Can't write mutations to Titan store " + tableName, ex); - throw new PermanentStorageException(ex); + throw new TemporaryStorageException(ex); } finally { try { writer.close(); diff --git a/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java b/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java index 88475d506c..49a4d5adcc 100644 --- a/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java +++ b/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java @@ -3,6 +3,7 @@ import com.thinkaurelius.titan.diskstorage.PermanentStorageException; import com.thinkaurelius.titan.diskstorage.StaticBuffer; import com.thinkaurelius.titan.diskstorage.StorageException; +import com.thinkaurelius.titan.diskstorage.TemporaryStorageException; import com.thinkaurelius.titan.diskstorage.common.DistributedStoreManager; import com.thinkaurelius.titan.diskstorage.keycolumnvalue.ConsistencyLevel; import com.thinkaurelius.titan.diskstorage.keycolumnvalue.Entry; @@ -176,7 +177,7 @@ public void clearStorage() throws StorageException { deleter.delete(); } catch (MutationsRejectedException ex) { logger.error("Can't write mutations to " + tableName, ex); - throw new PermanentStorageException(ex); + throw new TemporaryStorageException(ex); } finally { deleter.close(); } @@ -216,13 +217,13 @@ public void mutateMany(Map> mutations, St writer.flush(); } catch (MutationsRejectedException ex) { logger.error("Can't write mutations to Titan store " + tableName, ex); - throw new PermanentStorageException(ex); + throw new TemporaryStorageException(ex); } finally { try { writer.close(); } catch (MutationsRejectedException ex) { logger.error("Can't write mutations to Titan store " + tableName, ex); - throw new PermanentStorageException(ex); + throw new TemporaryStorageException(ex); } } } catch (TableNotFoundException ex) { diff --git a/titan-accumulo/titan-accumulo-iterators/pom.xml b/titan-accumulo/titan-accumulo-iterators/pom.xml index b2e896c4ef..4c8da85484 100644 --- a/titan-accumulo/titan-accumulo-iterators/pom.xml +++ b/titan-accumulo/titan-accumulo-iterators/pom.xml @@ -8,6 +8,7 @@ ../pom.xml titan-accumulo-iterators + Titan-Accumulo: Graph Database Iterators org.apache.accumulo diff --git a/titan-all/pom.xml b/titan-all/pom.xml index 68c1d9b25d..5548396f2c 100644 --- a/titan-all/pom.xml +++ b/titan-all/pom.xml @@ -41,8 +41,14 @@ com.thinkaurelius.titan - titan-accumulo + titan-accumulo-core ${titan.version} + + + zookeeper + org.apache.zookeeper + + com.thinkaurelius.titan diff --git a/titan-dist/pom.xml b/titan-dist/pom.xml index 8a26e6df92..e91632cb69 100644 --- a/titan-dist/pom.xml +++ b/titan-dist/pom.xml @@ -37,6 +37,7 @@ titan-dist-berkeleyje titan-dist-cassandra titan-dist-hbase + titan-dist-accumulo diff --git a/titan-dist/titan-dist-accumulo/assembly-accumulo.xml b/titan-dist/titan-dist-accumulo/assembly-accumulo.xml new file mode 100644 index 0000000000..53d5031876 --- /dev/null +++ b/titan-dist/titan-dist-accumulo/assembly-accumulo.xml @@ -0,0 +1,68 @@ + + accumulo-${project.version} + titan-accumulo-${project.version} + + + zip + tar.bz2 + + + + + ../src/assembly/skel + / + + + ../../config + + + ../../bin + bin + + hbase* + + + + + + + ../../CHANGELOG.textile + / + + + ../../LICENSE.txt + / + + + ../../NOTICE.txt + / + + + ../../UPGRADE.textile + / + + + + + + + /lib + false + runtime + false + + ${project.groupId}:titan-site:zip:htdocs:${project.version} + + + + + + true + /doc + false + + ${project.groupId}:titan-site:zip:htdocs:${project.version} + + + + diff --git a/titan-dist/titan-dist-accumulo/pom.xml b/titan-dist/titan-dist-accumulo/pom.xml new file mode 100644 index 0000000000..222162a9b1 --- /dev/null +++ b/titan-dist/titan-dist-accumulo/pom.xml @@ -0,0 +1,67 @@ + + 4.0.0 + + com.thinkaurelius.titan + titan-dist + 0.4.0-SNAPSHOT + ../pom.xml + + pom + titan-dist-accumulo + Titan-Dist-Accumulo: Archives with Accumulo + http://thinkaurelius.github.com/titan/ + + + assembly-accumulo.xml + + + + + titan-release + + + com.thinkaurelius.titan + titan-accumulo-core + ${project.version} + + + com.thinkaurelius.titan + titan-site + htdocs + ${project.version} + zip + + + + + + + maven-assembly-plugin + + + assemble-distribution-archive + + + + + maven-gpg-plugin + + + sign-artifacts + + + + + org.codehaus.mojo + wagon-maven-plugin + + + deploy-distribution-archives-to-s3 + + + + + + + + diff --git a/titan-dist/titan-dist-all/pom.xml b/titan-dist/titan-dist-all/pom.xml index f146bb5f6d..b2109a684c 100644 --- a/titan-dist/titan-dist-all/pom.xml +++ b/titan-dist/titan-dist-all/pom.xml @@ -52,6 +52,17 @@ titan-hbase ${project.version} + + com.thinkaurelius.titan + titan-accumulo-core + ${project.version} + + + zookeeper + org.apache.zookeeper + + + com.thinkaurelius.titan titan-lucene From de661b236b68dc6fba01f7627b0a50b127fe3420 Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Tue, 24 Sep 2013 09:59:19 -0400 Subject: [PATCH 31/50] Change executor service to immediate shut down in ConcurrentLists. --- .../titan/diskstorage/accumulo/util/ConcurrentLists.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/util/ConcurrentLists.java b/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/util/ConcurrentLists.java index 97f8973a95..3cda10c50f 100644 --- a/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/util/ConcurrentLists.java +++ b/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/util/ConcurrentLists.java @@ -58,7 +58,7 @@ public static List transform(List fromList, return transform(fromList, function, executor); } finally { if (executor != null) { - executor.shutdown(); + executor.shutdownNow(); } } } From e7998d80aa041a0f155ba778cac4951d1d507680 Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Thu, 26 Sep 2013 16:16:47 -0400 Subject: [PATCH 32/50] Expose AccumuloKeyColumnValueStore.getColumnSliceIterator(SliceQuery) for use in Faunus TitanAccumuloInputFormat. --- .../titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java b/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java index b25ba7a1dc..fb3b726367 100644 --- a/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java +++ b/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java @@ -355,7 +355,7 @@ private Range getRange(StaticBuffer key, SliceQuery query) { return new Range(startKey, true, endKey, false); } - private IteratorSetting getColumnSliceIterator(SliceQuery sliceQuery) { + public static IteratorSetting getColumnSliceIterator(SliceQuery sliceQuery) { IteratorSetting is = null; byte[] minColumn = sliceQuery.getSliceStart().as(StaticBuffer.ARRAY_FACTORY); From 09785f1e97d4adc7e239b751d93665fe4499289e Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Fri, 11 Oct 2013 12:37:36 -0400 Subject: [PATCH 33/50] Update Accumulo storage backend to current Titan master 0.4.0 snapshot, 2 failing unit tests. --- titan-accumulo/titan-accumulo-core/pom.xml | 57 +------------ .../accumulo/AccumuloKeyColumnValueStore.java | 27 +++--- .../accumulo/AccumuloStoreManager.java | 10 ++- .../accumulo/AccumuloTransaction.java | 8 +- ...> AccumuloGraphPerformanceMemoryTest.java} | 8 +- .../titan-accumulo-iterators/pom.xml | 83 ++----------------- 6 files changed, 40 insertions(+), 153 deletions(-) rename titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/graphdb/accumulo/{AccumuloGraphPerformanceTest.java => AccumuloGraphPerformanceMemoryTest.java} (62%) diff --git a/titan-accumulo/titan-accumulo-core/pom.xml b/titan-accumulo/titan-accumulo-core/pom.xml index 5caf5a2ca8..42d9ab2638 100644 --- a/titan-accumulo/titan-accumulo-core/pom.xml +++ b/titan-accumulo/titan-accumulo-core/pom.xml @@ -64,49 +64,16 @@ 0.6.1 - - - comprehensive - - - - maven-surefire-plugin - 2.12.1 - - -Xms256m -Xmx4G - - **/*Test.java - - - **/Internal*.java - **/*Suite.java - - always - - - - - - - ${basedir}/target - ${project.artifactId}-${project.version} ${basedir}/src/main/resources true - - - ${basedir}/src/test/resources - - - maven-dependency-plugin - 2.7 test-compile @@ -114,6 +81,7 @@ copy-dependencies + com.thinkaurelius.titan target/test-lib false false @@ -122,29 +90,6 @@ - - maven-surefire-plugin - - -Xms256m -Xmx1G ${jacoco.opts} - - - **/*PerformanceTest.java - **/*ConcurrentTest.java - - - - - - maven-resources-plugin - - - org.jacoco - jacoco-maven-plugin - - - maven-antrun-plugin - - \ No newline at end of file diff --git a/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java b/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java index fb3b726367..c17f82d727 100644 --- a/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java +++ b/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java @@ -244,11 +244,6 @@ public StaticBuffer[] getLocalKeyPartition() throws StorageException { throw new UnsupportedOperationException(); // Accumulo stores do not support local key partitions. } - @Override - public RecordIterator getKeys(StoreTransaction txh) throws StorageException { - return executeKeySliceQuery(null, true); - } - @Override public KeyIterator getKeys(KeyRangeQuery query, StoreTransaction txh) throws StorageException { return executeKeySliceQuery(query.getKeyStart(), query.getKeyEnd(), query, false); @@ -384,46 +379,56 @@ private static class RowKeyIterator implements KeyIterator { public RecordIterator getEntries() { RecordIterator rowIter = new RecordIterator() { @Override - public boolean hasNext() throws StorageException { + public boolean hasNext() { ensureOpen(); return currentRow.hasNext(); } @Override - public Entry next() throws StorageException { + public Entry next() { ensureOpen(); Map.Entry kv = currentRow.next(); return getEntry(kv); } @Override - public void close() throws StorageException { + public void close() { isClosed = true; // same semantics as in-memory implementation in Titan core } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } }; return rowIter; } @Override - public boolean hasNext() throws StorageException { + public boolean hasNext() { ensureOpen(); return rows.hasNext(); } @Override - public StaticBuffer next() throws StorageException { + public StaticBuffer next() { ensureOpen(); currentRow = Iterators.peekingIterator(rows.next()); return new StaticArrayBuffer(currentRow.peek().getKey().getRow().getBytes()); } @Override - public void close() throws StorageException { + public void close() { isClosed = true; rows = null; currentRow = null; } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } private void ensureOpen() { if (isClosed) { diff --git a/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java b/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java index 49a4d5adcc..e9cd5be895 100644 --- a/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java +++ b/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java @@ -12,6 +12,7 @@ import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyColumnValueStoreManager; import com.thinkaurelius.titan.diskstorage.keycolumnvalue.StoreFeatures; import com.thinkaurelius.titan.diskstorage.keycolumnvalue.StoreTransaction; +import com.thinkaurelius.titan.diskstorage.keycolumnvalue.StoreTxConfig; import com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration; import java.util.Collection; import java.util.Collections; @@ -107,7 +108,8 @@ public AccumuloStoreManager(Configuration config) throws StorageException { openStores = new ConcurrentHashMap(); features = new StoreFeatures(); - features.supportsScan = true; + features.supportsOrderedScan = true; + features.supportsUnorderedScan = true; features.supportsBatchMutation = true; features.supportsTransactions = false; features.supportsConsistentKeyOperations = true; @@ -197,10 +199,10 @@ public String getConfigurationProperty(String key) throws StorageException { public void setConfigurationProperty(String key, String value) throws StorageException { storeConfiguration.setConfigurationProperty(key, value); } - + @Override - public StoreTransaction beginTransaction(ConsistencyLevel level) throws StorageException { - return new AccumuloTransaction(level); + public StoreTransaction beginTransaction(final StoreTxConfig config) throws StorageException { + return new AccumuloTransaction(config); } @Override diff --git a/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloTransaction.java b/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloTransaction.java index 1f3c24e53f..0b9a17a7fe 100644 --- a/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloTransaction.java +++ b/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloTransaction.java @@ -1,7 +1,7 @@ package com.thinkaurelius.titan.diskstorage.accumulo; import com.thinkaurelius.titan.diskstorage.common.AbstractStoreTransaction; -import com.thinkaurelius.titan.diskstorage.keycolumnvalue.ConsistencyLevel; +import com.thinkaurelius.titan.diskstorage.keycolumnvalue.StoreTxConfig; /** * This creates a transaction type specific to Accumulo. @@ -9,8 +9,8 @@ * @author Etienne Deprit */ public class AccumuloTransaction extends AbstractStoreTransaction { - - public AccumuloTransaction(ConsistencyLevel level) { - super(ConsistencyLevel.KEY_CONSISTENT); + + public AccumuloTransaction(final StoreTxConfig config) { + super(config); } } diff --git a/titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/graphdb/accumulo/AccumuloGraphPerformanceTest.java b/titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/graphdb/accumulo/AccumuloGraphPerformanceMemoryTest.java similarity index 62% rename from titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/graphdb/accumulo/AccumuloGraphPerformanceTest.java rename to titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/graphdb/accumulo/AccumuloGraphPerformanceMemoryTest.java index 88787ecbfd..7b475b968c 100644 --- a/titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/graphdb/accumulo/AccumuloGraphPerformanceTest.java +++ b/titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/graphdb/accumulo/AccumuloGraphPerformanceMemoryTest.java @@ -1,18 +1,18 @@ package com.thinkaurelius.titan.graphdb.accumulo; import com.thinkaurelius.titan.AccumuloStorageSetup; -import com.thinkaurelius.titan.graphdb.TitanGraphPerformanceTest; +import com.thinkaurelius.titan.graphdb.TitanGraphPerformanceMemoryTest; import org.junit.BeforeClass; import java.io.IOException; -public class AccumuloGraphPerformanceTest extends TitanGraphPerformanceTest { +public class AccumuloGraphPerformanceMemoryTest extends TitanGraphPerformanceMemoryTest { @BeforeClass public static void startAccumulo() throws IOException { AccumuloStorageSetup.startAccumulo(); } - public AccumuloGraphPerformanceTest() { - super(AccumuloStorageSetup.getAccumuloGraphConfiguration(), 0, 1, false); + public AccumuloGraphPerformanceMemoryTest() { + super(AccumuloStorageSetup.getAccumuloGraphConfiguration()); } } diff --git a/titan-accumulo/titan-accumulo-iterators/pom.xml b/titan-accumulo/titan-accumulo-iterators/pom.xml index 4c8da85484..64937a720d 100644 --- a/titan-accumulo/titan-accumulo-iterators/pom.xml +++ b/titan-accumulo/titan-accumulo-iterators/pom.xml @@ -40,6 +40,13 @@ 1.0.4 provided + + com.thinkaurelius.titan + titan-test + ${project.version} + test + ${titan.classifier} + junit junit @@ -47,66 +54,16 @@ provided - - - comprehensive - - - - maven-surefire-plugin - 2.12.1 - - -Xms256m -Xmx4G - - **/*Test.java - - - **/Internal*.java - **/*Suite.java - - always - - - - - - - ${basedir}/target - ${project.artifactId}-${project.version} ${basedir}/src/main/resources true - - - ${basedir}/src/test/resources - - - - - org.apache.maven.plugins - maven-shade-plugin - 2.0 - - - package - - shade - - - - - false - true - - maven-dependency-plugin - 2.7 test-compile @@ -114,6 +71,7 @@ copy-dependencies + com.thinkaurelius.titan target/test-lib false false @@ -122,29 +80,6 @@ - - maven-surefire-plugin - - -Xms256m -Xmx1G ${jacoco.opts} - - - **/*PerformanceTest.java - **/*ConcurrentTest.java - - - - - - maven-resources-plugin - - - org.jacoco - jacoco-maven-plugin - - - maven-antrun-plugin - - - \ No newline at end of file + From 9adaed62278d6b9b776ef1455ff0b27bf63fa868 Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Fri, 11 Oct 2013 14:34:31 -0400 Subject: [PATCH 34/50] Fixed scanner range in store getKeys, unit tests now pass. --- .../accumulo/AccumuloKeyColumnValueStore.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java b/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java index c17f82d727..e05a731ca3 100644 --- a/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java +++ b/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloKeyColumnValueStore.java @@ -213,7 +213,8 @@ private static Mutation makeDeleteMutation(Text colFamily, Text key, List Date: Fri, 11 Oct 2013 15:15:42 -0400 Subject: [PATCH 35/50] Added titan-accumulo to jre7 profile. --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 0dfe377418..0ebf45b864 100644 --- a/pom.xml +++ b/pom.xml @@ -717,6 +717,7 @@ titan-cassandra titan-persistit titan-hbase + titan-accumulo titan-es titan-lucene titan-hazelcast From 765749013adbeb44b90448a593d6f3ea28e5286b Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Fri, 18 Oct 2013 10:38:37 -0400 Subject: [PATCH 36/50] Update build files for Titan 0.4.0 release. --- pom.xml | 1 + titan-accumulo/pom.xml | 4 ++-- titan-accumulo/titan-accumulo-core/pom.xml | 4 ++-- titan-accumulo/titan-accumulo-iterators/pom.xml | 2 +- titan-dist/titan-dist-accumulo/pom.xml | 2 +- 5 files changed, 7 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index 4d0d71c5bc..31eedf50b7 100644 --- a/pom.xml +++ b/pom.xml @@ -91,6 +91,7 @@ titan-berkeleyje titan-cassandra titan-hbase + titan-accumulo titan-es titan-lucene titan-hazelcast diff --git a/titan-accumulo/pom.xml b/titan-accumulo/pom.xml index 357420d026..9eaa503727 100644 --- a/titan-accumulo/pom.xml +++ b/titan-accumulo/pom.xml @@ -4,7 +4,7 @@ com.thinkaurelius.titan titan - 0.4.0-SNAPSHOT + 0.4.0 ../pom.xml titan-accumulo @@ -16,4 +16,4 @@ titan-accumulo-iterators titan-accumulo-core - \ No newline at end of file + diff --git a/titan-accumulo/titan-accumulo-core/pom.xml b/titan-accumulo/titan-accumulo-core/pom.xml index 42d9ab2638..8205d37933 100644 --- a/titan-accumulo/titan-accumulo-core/pom.xml +++ b/titan-accumulo/titan-accumulo-core/pom.xml @@ -4,7 +4,7 @@ com.thinkaurelius.titan titan-accumulo - 0.4.0-SNAPSHOT + 0.4.0 ../pom.xml titan-accumulo-core @@ -92,4 +92,4 @@ - \ No newline at end of file + diff --git a/titan-accumulo/titan-accumulo-iterators/pom.xml b/titan-accumulo/titan-accumulo-iterators/pom.xml index 64937a720d..4a46d546ad 100644 --- a/titan-accumulo/titan-accumulo-iterators/pom.xml +++ b/titan-accumulo/titan-accumulo-iterators/pom.xml @@ -4,7 +4,7 @@ com.thinkaurelius.titan titan-accumulo - 0.4.0-SNAPSHOT + 0.4.0 ../pom.xml titan-accumulo-iterators diff --git a/titan-dist/titan-dist-accumulo/pom.xml b/titan-dist/titan-dist-accumulo/pom.xml index 222162a9b1..d8351f3c1c 100644 --- a/titan-dist/titan-dist-accumulo/pom.xml +++ b/titan-dist/titan-dist-accumulo/pom.xml @@ -3,7 +3,7 @@ com.thinkaurelius.titan titan-dist - 0.4.0-SNAPSHOT + 0.4.0 ../pom.xml pom From cb1394148899cab710a7d90a05f1831b694d7490 Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Fri, 1 Nov 2013 11:45:00 -0400 Subject: [PATCH 37/50] Updated version to 0.4.1-SNAPSHOT. --- titan-accumulo/pom.xml | 2 +- titan-accumulo/titan-accumulo-core/pom.xml | 2 +- titan-accumulo/titan-accumulo-iterators/pom.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/titan-accumulo/pom.xml b/titan-accumulo/pom.xml index 9eaa503727..29443625f1 100644 --- a/titan-accumulo/pom.xml +++ b/titan-accumulo/pom.xml @@ -4,7 +4,7 @@ com.thinkaurelius.titan titan - 0.4.0 + 0.4.1-SNAPSHOT ../pom.xml titan-accumulo diff --git a/titan-accumulo/titan-accumulo-core/pom.xml b/titan-accumulo/titan-accumulo-core/pom.xml index 8205d37933..fc70695a8a 100644 --- a/titan-accumulo/titan-accumulo-core/pom.xml +++ b/titan-accumulo/titan-accumulo-core/pom.xml @@ -4,7 +4,7 @@ com.thinkaurelius.titan titan-accumulo - 0.4.0 + 0.4.1-SNAPSHOT ../pom.xml titan-accumulo-core diff --git a/titan-accumulo/titan-accumulo-iterators/pom.xml b/titan-accumulo/titan-accumulo-iterators/pom.xml index 4a46d546ad..e109beaa44 100644 --- a/titan-accumulo/titan-accumulo-iterators/pom.xml +++ b/titan-accumulo/titan-accumulo-iterators/pom.xml @@ -4,7 +4,7 @@ com.thinkaurelius.titan titan-accumulo - 0.4.0 + 0.4.1-SNAPSHOT ../pom.xml titan-accumulo-iterators From 4ea0e018e5ef1ad05841a55b3a5be1d479b2a520 Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Fri, 1 Nov 2013 11:54:02 -0400 Subject: [PATCH 38/50] Updated Titan-Dist-Accumulo to version 0.4.1-SNAPSHOT. --- titan-dist/titan-dist-accumulo/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/titan-dist/titan-dist-accumulo/pom.xml b/titan-dist/titan-dist-accumulo/pom.xml index d8351f3c1c..a84c890f1b 100644 --- a/titan-dist/titan-dist-accumulo/pom.xml +++ b/titan-dist/titan-dist-accumulo/pom.xml @@ -3,7 +3,7 @@ com.thinkaurelius.titan titan-dist - 0.4.0 + 0.4.1-SNAPSHOT ../pom.xml pom From e6ccabeadd86f42401926b72d20fd291071cc466 Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Mon, 4 Nov 2013 12:50:30 -0500 Subject: [PATCH 39/50] Added supportsMultiQuery = true to Accumulo features. --- .../titan/diskstorage/accumulo/AccumuloStoreManager.java | 1 + 1 file changed, 1 insertion(+) diff --git a/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java b/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java index e9cd5be895..54ce02fd35 100644 --- a/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java +++ b/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java @@ -112,6 +112,7 @@ public AccumuloStoreManager(Configuration config) throws StorageException { features.supportsUnorderedScan = true; features.supportsBatchMutation = true; features.supportsTransactions = false; + features.supportsMultiQuery = true; features.supportsConsistentKeyOperations = true; features.supportsLocking = false; features.isKeyOrdered = false; From cb5fd0b9e82445abe9b51a4bd9f1a4dffb9b2ae9 Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Tue, 12 Nov 2013 18:35:20 -0500 Subject: [PATCH 40/50] Added Accumulo ID allocation test. --- .../accumulo/AccumuloIDAllocationTest.java | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloIDAllocationTest.java diff --git a/titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloIDAllocationTest.java b/titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloIDAllocationTest.java new file mode 100644 index 0000000000..caf2682b9c --- /dev/null +++ b/titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloIDAllocationTest.java @@ -0,0 +1,27 @@ +package com.thinkaurelius.titan.diskstorage.accumulo; + +import com.thinkaurelius.titan.AccumuloStorageSetup; +import com.thinkaurelius.titan.diskstorage.IDAllocationTest; +import com.thinkaurelius.titan.diskstorage.LockKeyColumnValueStoreTest; +import com.thinkaurelius.titan.diskstorage.StorageException; +import com.thinkaurelius.titan.diskstorage.keycolumnvalue.KeyColumnValueStoreManager; +import org.apache.commons.configuration.Configuration; +import org.junit.BeforeClass; + +import java.io.IOException; + +public class AccumuloIDAllocationTest extends IDAllocationTest { + + public AccumuloIDAllocationTest(Configuration baseConfig) { + super(baseConfig); + } + + @BeforeClass + public static void startAccmulo() throws IOException { + AccumuloStorageSetup.startAccumulo(); + } + + public KeyColumnValueStoreManager openStorageManager(int idx) throws StorageException { + return AccumuloStorageSetup.getAccumuloStoreManager(); + } +} From cb23da555a2c0ea871e77bce129a35dc98a0529a Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Tue, 12 Nov 2013 18:45:49 -0500 Subject: [PATCH 41/50] Accumulo username/password configuration moved to DistributedStoreManager. --- .../titan/diskstorage/accumulo/AccumuloStoreManager.java | 9 --------- .../com/thinkaurelius/titan/AccumuloStorageSetup.java | 7 +++---- 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java b/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java index 54ce02fd35..b063406235 100644 --- a/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java +++ b/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java @@ -53,8 +53,6 @@ public class AccumuloStoreManager extends DistributedStoreManager implements Key public static final String ACCUMULO_CONFIGURATION_NAMESPACE = "accumulo-config"; // Configuration keys public static final String ACCUMULO_INTSANCE_KEY = "instance"; - public static final String ACCUMULO_USER_KEY = "username"; - public static final String ACCUMULO_PASSWORD_KEY = "password"; public static final String TABLE_NAME_KEY = "tablename"; public static final String SERVER_SIDE_ITERATORS_KEY = "server-side-iterators"; // Configuration defaults @@ -67,8 +65,6 @@ public class AccumuloStoreManager extends DistributedStoreManager implements Key private final String tableName; private final String instanceName; private final String zooKeepers; - private final String username; - private final String password; private final boolean serverSideIterators; private final Instance instance; // thread-safe private final Connector connector; // thread-safe @@ -88,9 +84,6 @@ public AccumuloStoreManager(Configuration config) throws StorageException { Configuration accumuloConfig = config.subset(ACCUMULO_CONFIGURATION_NAMESPACE); instanceName = accumuloConfig.getString(ACCUMULO_INTSANCE_KEY); - username = accumuloConfig.getString(ACCUMULO_USER_KEY); - password = accumuloConfig.getString(ACCUMULO_PASSWORD_KEY); - serverSideIterators = accumuloConfig.getBoolean(SERVER_SIDE_ITERATORS_KEY, SERVER_SIDE_ITERATORS_DEFAULT); instance = instanceFactory.getInstance(instanceName, zooKeepers); @@ -191,12 +184,10 @@ public void clearStorage() throws StorageException { } } - @Override public String getConfigurationProperty(String key) throws StorageException { return storeConfiguration.getConfigurationProperty(key); } - @Override public void setConfigurationProperty(String key, String value) throws StorageException { storeConfiguration.setConfigurationProperty(key, value); } diff --git a/titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java b/titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java index babaa8c7a9..42907b6d80 100644 --- a/titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java +++ b/titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java @@ -43,14 +43,13 @@ public static Configuration getAccumuloGraphConfiguration() { storageConfig.addProperty(GraphDatabaseConfiguration.STORAGE_BACKEND_KEY, "com.thinkaurelius.titan.diskstorage.accumulo.MockAccumuloStoreManager"); storageConfig.addProperty(GraphDatabaseConfiguration.HOSTNAME_KEY, "localhost"); + + storageConfig.addProperty(GraphDatabaseConfiguration.AUTH_USERNAME_KEY, "root"); + storageConfig.addProperty(GraphDatabaseConfiguration.AUTH_PASSWORD_KEY, ""); Configuration accumuloConfig = storageConfig.subset(AccumuloStoreManager.ACCUMULO_CONFIGURATION_NAMESPACE); accumuloConfig.addProperty(AccumuloStoreManager.ACCUMULO_INTSANCE_KEY, "devdb"); - - accumuloConfig.addProperty(AccumuloStoreManager.ACCUMULO_USER_KEY, "root"); - accumuloConfig.addProperty(AccumuloStoreManager.ACCUMULO_PASSWORD_KEY, ""); - accumuloConfig.addProperty(AccumuloStoreManager.SERVER_SIDE_ITERATORS_KEY, false); return config; From 9bda6f8cfd2cb85179e6e0b65072254fac00558c Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Mon, 2 Dec 2013 14:20:14 -0500 Subject: [PATCH 42/50] Move up Titan Accumulo version to 0.4.1 tag. --- titan-accumulo/pom.xml | 2 +- titan-accumulo/titan-accumulo-core/pom.xml | 2 +- titan-accumulo/titan-accumulo-iterators/pom.xml | 2 +- titan-dist/titan-dist-accumulo/pom.xml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/titan-accumulo/pom.xml b/titan-accumulo/pom.xml index 29443625f1..b14cf84148 100644 --- a/titan-accumulo/pom.xml +++ b/titan-accumulo/pom.xml @@ -4,7 +4,7 @@ com.thinkaurelius.titan titan - 0.4.1-SNAPSHOT + 0.4.1 ../pom.xml titan-accumulo diff --git a/titan-accumulo/titan-accumulo-core/pom.xml b/titan-accumulo/titan-accumulo-core/pom.xml index fc70695a8a..7de4709ef2 100644 --- a/titan-accumulo/titan-accumulo-core/pom.xml +++ b/titan-accumulo/titan-accumulo-core/pom.xml @@ -4,7 +4,7 @@ com.thinkaurelius.titan titan-accumulo - 0.4.1-SNAPSHOT + 0.4.1 ../pom.xml titan-accumulo-core diff --git a/titan-accumulo/titan-accumulo-iterators/pom.xml b/titan-accumulo/titan-accumulo-iterators/pom.xml index da9316fde6..172b65ac2a 100644 --- a/titan-accumulo/titan-accumulo-iterators/pom.xml +++ b/titan-accumulo/titan-accumulo-iterators/pom.xml @@ -4,7 +4,7 @@ com.thinkaurelius.titan titan-accumulo - 0.4.1-SNAPSHOT + 0.4.1 ../pom.xml titan-accumulo-iterators diff --git a/titan-dist/titan-dist-accumulo/pom.xml b/titan-dist/titan-dist-accumulo/pom.xml index a84c890f1b..dccf52eba5 100644 --- a/titan-dist/titan-dist-accumulo/pom.xml +++ b/titan-dist/titan-dist-accumulo/pom.xml @@ -3,7 +3,7 @@ com.thinkaurelius.titan titan-dist - 0.4.1-SNAPSHOT + 0.4.1 ../pom.xml pom From 355c62d0128622e61f7d551b83c078c3705e7061 Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Mon, 2 Dec 2013 14:26:21 -0500 Subject: [PATCH 43/50] Move up Titan Accumulo version to 0.4.2-SNAPSHOT. --- titan-accumulo/pom.xml | 2 +- titan-accumulo/titan-accumulo-core/pom.xml | 2 +- titan-accumulo/titan-accumulo-iterators/pom.xml | 2 +- titan-dist/titan-dist-accumulo/pom.xml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/titan-accumulo/pom.xml b/titan-accumulo/pom.xml index b14cf84148..28062eaf66 100644 --- a/titan-accumulo/pom.xml +++ b/titan-accumulo/pom.xml @@ -4,7 +4,7 @@ com.thinkaurelius.titan titan - 0.4.1 + 0.4.2-SNAPSHOT ../pom.xml titan-accumulo diff --git a/titan-accumulo/titan-accumulo-core/pom.xml b/titan-accumulo/titan-accumulo-core/pom.xml index 7de4709ef2..67a22b54bb 100644 --- a/titan-accumulo/titan-accumulo-core/pom.xml +++ b/titan-accumulo/titan-accumulo-core/pom.xml @@ -4,7 +4,7 @@ com.thinkaurelius.titan titan-accumulo - 0.4.1 + 0.4.2-SNAPSHOT ../pom.xml titan-accumulo-core diff --git a/titan-accumulo/titan-accumulo-iterators/pom.xml b/titan-accumulo/titan-accumulo-iterators/pom.xml index 172b65ac2a..c39f5d06a7 100644 --- a/titan-accumulo/titan-accumulo-iterators/pom.xml +++ b/titan-accumulo/titan-accumulo-iterators/pom.xml @@ -4,7 +4,7 @@ com.thinkaurelius.titan titan-accumulo - 0.4.1 + 0.4.2-SNAPSHOT ../pom.xml titan-accumulo-iterators diff --git a/titan-dist/titan-dist-accumulo/pom.xml b/titan-dist/titan-dist-accumulo/pom.xml index dccf52eba5..8af201196a 100644 --- a/titan-dist/titan-dist-accumulo/pom.xml +++ b/titan-dist/titan-dist-accumulo/pom.xml @@ -3,7 +3,7 @@ com.thinkaurelius.titan titan-dist - 0.4.1 + 0.4.2-SNAPSHOT ../pom.xml pom From fbce20b53a739b18ae59d09835693bc43fc9ba98 Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Tue, 3 Dec 2013 12:59:01 -0500 Subject: [PATCH 44/50] Added titan-test module as test dependency for titan-accumulo-iterators. --- titan-accumulo/titan-accumulo-iterators/pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/titan-accumulo/titan-accumulo-iterators/pom.xml b/titan-accumulo/titan-accumulo-iterators/pom.xml index 172b65ac2a..c8c347c3a2 100644 --- a/titan-accumulo/titan-accumulo-iterators/pom.xml +++ b/titan-accumulo/titan-accumulo-iterators/pom.xml @@ -46,6 +46,12 @@ 4.11 provided + + com.thinkaurelius.titan + titan-test + ${project.version} + test + From 8bc92e6b17e43e89c4d5001421dd5ac42621cf3e Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Mon, 6 Jan 2014 13:47:26 -0500 Subject: [PATCH 45/50] Corrected Accumulo Titan distro project so it mimics HBase distro. --- titan-dist/titan-dist-accumulo/accumulo.xml | 37 ++++++++++ .../titan-dist-accumulo/assembly-accumulo.xml | 68 ------------------- titan-dist/titan-dist-accumulo/pom.xml | 49 +++---------- 3 files changed, 45 insertions(+), 109 deletions(-) create mode 100644 titan-dist/titan-dist-accumulo/accumulo.xml delete mode 100644 titan-dist/titan-dist-accumulo/assembly-accumulo.xml diff --git a/titan-dist/titan-dist-accumulo/accumulo.xml b/titan-dist/titan-dist-accumulo/accumulo.xml new file mode 100644 index 0000000000..5e57bcff79 --- /dev/null +++ b/titan-dist/titan-dist-accumulo/accumulo.xml @@ -0,0 +1,37 @@ + + + ${distribution.assembly.name}-${project.version} + titan-${distribution.assembly.name}-${project.version} + + + zip + tar.bz2 + + + + ../src/assembly/descriptor/common.component.xml + ../src/assembly/descriptor/readmes.component.xml + ../src/assembly/descriptor/htmldocs.component.xml + + + + + 775 + 775 + ../../titan-accumulo/bin + /bin + + + ../../titan-accumulo/config + /conf + + + ../../conf + /conf + + titan-*${distribution.assembly.name}* + + + + diff --git a/titan-dist/titan-dist-accumulo/assembly-accumulo.xml b/titan-dist/titan-dist-accumulo/assembly-accumulo.xml deleted file mode 100644 index 53d5031876..0000000000 --- a/titan-dist/titan-dist-accumulo/assembly-accumulo.xml +++ /dev/null @@ -1,68 +0,0 @@ - - accumulo-${project.version} - titan-accumulo-${project.version} - - - zip - tar.bz2 - - - - - ../src/assembly/skel - / - - - ../../config - - - ../../bin - bin - - hbase* - - - - - - - ../../CHANGELOG.textile - / - - - ../../LICENSE.txt - / - - - ../../NOTICE.txt - / - - - ../../UPGRADE.textile - / - - - - - - - /lib - false - runtime - false - - ${project.groupId}:titan-site:zip:htdocs:${project.version} - - - - - - true - /doc - false - - ${project.groupId}:titan-site:zip:htdocs:${project.version} - - - - diff --git a/titan-dist/titan-dist-accumulo/pom.xml b/titan-dist/titan-dist-accumulo/pom.xml index dccf52eba5..c8ea45ff45 100644 --- a/titan-dist/titan-dist-accumulo/pom.xml +++ b/titan-dist/titan-dist-accumulo/pom.xml @@ -2,9 +2,9 @@ 4.0.0 com.thinkaurelius.titan - titan-dist + titan-dist-parent 0.4.1 - ../pom.xml + ../titan-dist-parent/pom.xml pom titan-dist-accumulo @@ -12,56 +12,23 @@ http://thinkaurelius.github.com/titan/ - assembly-accumulo.xml + accumulo + ${project.basedir}/../src/assembly/descriptor/backend.xml + titan-accumulo*.properties + ${basedir}/../.. - titan-release + aurelius-release + com.thinkaurelius.titan titan-accumulo-core ${project.version} - - com.thinkaurelius.titan - titan-site - htdocs - ${project.version} - zip - - - - - - maven-assembly-plugin - - - assemble-distribution-archive - - - - - maven-gpg-plugin - - - sign-artifacts - - - - - org.codehaus.mojo - wagon-maven-plugin - - - deploy-distribution-archives-to-s3 - - - - - From 1cdc8a6f31382cac9f696bdf7bee04cfaefa1070 Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Mon, 6 Jan 2014 18:29:34 -0500 Subject: [PATCH 46/50] Update Accumulo to version 1.4.4. --- titan-accumulo/pom.xml | 4 ++++ titan-accumulo/titan-accumulo-core/pom.xml | 6 +++--- titan-accumulo/titan-accumulo-iterators/pom.xml | 4 ++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/titan-accumulo/pom.xml b/titan-accumulo/pom.xml index b14cf84148..1da3180657 100644 --- a/titan-accumulo/pom.xml +++ b/titan-accumulo/pom.xml @@ -12,6 +12,10 @@ Titan-Accumulo: Distributed Graph Database http://thinkaurelius.github.com/titan/ + + 1.4.4 + + titan-accumulo-iterators titan-accumulo-core diff --git a/titan-accumulo/titan-accumulo-core/pom.xml b/titan-accumulo/titan-accumulo-core/pom.xml index 7de4709ef2..69c624bd64 100644 --- a/titan-accumulo/titan-accumulo-core/pom.xml +++ b/titan-accumulo/titan-accumulo-core/pom.xml @@ -29,7 +29,7 @@ org.apache.accumulo accumulo-core - 1.4.3 + ${accumulo.version} libthrift @@ -40,7 +40,7 @@ org.apache.accumulo accumulo-start - 1.4.3 + ${accumulo.version} commons-logging-api @@ -51,7 +51,7 @@ org.apache.zookeeper zookeeper - 3.3.6 + 3.4.5 org.apache.hadoop diff --git a/titan-accumulo/titan-accumulo-iterators/pom.xml b/titan-accumulo/titan-accumulo-iterators/pom.xml index c8c347c3a2..1099b581eb 100644 --- a/titan-accumulo/titan-accumulo-iterators/pom.xml +++ b/titan-accumulo/titan-accumulo-iterators/pom.xml @@ -13,7 +13,7 @@ org.apache.accumulo accumulo-core - 1.4.3 + ${accumulo.version} provided @@ -25,7 +25,7 @@ org.apache.accumulo accumulo-start - 1.4.3 + ${accumulo.version} provided From ba00005495aafe7c090bc61917d5d123f8f9f392 Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Mon, 6 Jan 2014 18:30:59 -0500 Subject: [PATCH 47/50] Force Titan Accumulo distro to use Thrift 0.6.1. --- titan-dist/pom.xml | 28 +++++++++++++------------- titan-dist/titan-dist-accumulo/pom.xml | 11 ++++++++++ titan-dist/titan-dist-all/pom.xml | 6 ------ 3 files changed, 25 insertions(+), 20 deletions(-) diff --git a/titan-dist/pom.xml b/titan-dist/pom.xml index 21a32085cc..d8c7b9466f 100644 --- a/titan-dist/pom.xml +++ b/titan-dist/pom.xml @@ -165,26 +165,26 @@ compile-tests - test-compile - - testCompile - - - - + test-compile + + testCompile + + + + maven-jar-plugin pack-test-jar - package - - test-jar - - - - + package + + test-jar + + + + diff --git a/titan-dist/titan-dist-accumulo/pom.xml b/titan-dist/titan-dist-accumulo/pom.xml index c8ea45ff45..ea6f8d11bb 100644 --- a/titan-dist/titan-dist-accumulo/pom.xml +++ b/titan-dist/titan-dist-accumulo/pom.xml @@ -27,6 +27,17 @@ com.thinkaurelius.titan titan-accumulo-core ${project.version} + + + libthrift + org.apache.thrift + + + + + org.apache.thrift + libthrift + 0.6.1 diff --git a/titan-dist/titan-dist-all/pom.xml b/titan-dist/titan-dist-all/pom.xml index 1e0c702891..f0a39a3edd 100644 --- a/titan-dist/titan-dist-all/pom.xml +++ b/titan-dist/titan-dist-all/pom.xml @@ -51,12 +51,6 @@ com.thinkaurelius.titan titan-accumulo-core ${project.version} - - - zookeeper - org.apache.zookeeper - - com.thinkaurelius.titan From 786c197f1373da3214766b3909dd94fd9eab87c3 Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Mon, 13 Jan 2014 14:18:43 -0500 Subject: [PATCH 48/50] Move Titan version to 0.4.2. --- titan-accumulo/pom.xml | 2 +- titan-accumulo/titan-accumulo-core/pom.xml | 2 +- titan-accumulo/titan-accumulo-iterators/pom.xml | 2 +- titan-dist/titan-dist-accumulo/pom.xml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/titan-accumulo/pom.xml b/titan-accumulo/pom.xml index 1da3180657..168b00267c 100644 --- a/titan-accumulo/pom.xml +++ b/titan-accumulo/pom.xml @@ -4,7 +4,7 @@ com.thinkaurelius.titan titan - 0.4.1 + 0.4.2 ../pom.xml titan-accumulo diff --git a/titan-accumulo/titan-accumulo-core/pom.xml b/titan-accumulo/titan-accumulo-core/pom.xml index 69c624bd64..4ea734e57e 100644 --- a/titan-accumulo/titan-accumulo-core/pom.xml +++ b/titan-accumulo/titan-accumulo-core/pom.xml @@ -4,7 +4,7 @@ com.thinkaurelius.titan titan-accumulo - 0.4.1 + 0.4.2 ../pom.xml titan-accumulo-core diff --git a/titan-accumulo/titan-accumulo-iterators/pom.xml b/titan-accumulo/titan-accumulo-iterators/pom.xml index 1099b581eb..9c57d3f945 100644 --- a/titan-accumulo/titan-accumulo-iterators/pom.xml +++ b/titan-accumulo/titan-accumulo-iterators/pom.xml @@ -4,7 +4,7 @@ com.thinkaurelius.titan titan-accumulo - 0.4.1 + 0.4.2 ../pom.xml titan-accumulo-iterators diff --git a/titan-dist/titan-dist-accumulo/pom.xml b/titan-dist/titan-dist-accumulo/pom.xml index ea6f8d11bb..0a70fcba28 100644 --- a/titan-dist/titan-dist-accumulo/pom.xml +++ b/titan-dist/titan-dist-accumulo/pom.xml @@ -3,7 +3,7 @@ com.thinkaurelius.titan titan-dist-parent - 0.4.1 + 0.4.2 ../titan-dist-parent/pom.xml pom From 2878f6b90f607c855ea3e3f26727c4ed4a5f0ed0 Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Tue, 14 Jan 2014 22:58:21 -0500 Subject: [PATCH 49/50] Shorten Accumulo configuration namespace key. --- .../titan/diskstorage/accumulo/AccumuloStoreManager.java | 6 +++--- .../java/com/thinkaurelius/titan/AccumuloStorageSetup.java | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java b/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java index b063406235..edf99cd615 100644 --- a/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java +++ b/titan-accumulo/titan-accumulo-core/src/main/java/com/thinkaurelius/titan/diskstorage/accumulo/AccumuloStoreManager.java @@ -50,7 +50,7 @@ public class AccumuloStoreManager extends DistributedStoreManager implements Key // Default parameters private static final Authorizations AUTHORIZATIONS_DEFAULT = new Authorizations(); // Configuration namespace - public static final String ACCUMULO_CONFIGURATION_NAMESPACE = "accumulo-config"; + public static final String ACCUMULO_NAMESPACE = "accumulo"; // Configuration keys public static final String ACCUMULO_INTSANCE_KEY = "instance"; public static final String TABLE_NAME_KEY = "tablename"; @@ -81,7 +81,7 @@ public AccumuloStoreManager(Configuration config) throws StorageException { tableName = config.getString(TABLE_NAME_KEY, TABLE_NAME_DEFAULT); // Accumulo specific keys - Configuration accumuloConfig = config.subset(ACCUMULO_CONFIGURATION_NAMESPACE); + Configuration accumuloConfig = config.subset(ACCUMULO_NAMESPACE); instanceName = accumuloConfig.getString(ACCUMULO_INTSANCE_KEY); serverSideIterators = accumuloConfig.getBoolean(SERVER_SIDE_ITERATORS_KEY, SERVER_SIDE_ITERATORS_DEFAULT); @@ -108,7 +108,7 @@ public AccumuloStoreManager(Configuration config) throws StorageException { features.supportsMultiQuery = true; features.supportsConsistentKeyOperations = true; features.supportsLocking = false; - features.isKeyOrdered = false; + features.isKeyOrdered = true; features.isDistributed = true; features.hasLocalKeyPartition = false; diff --git a/titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java b/titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java index 42907b6d80..f37c387798 100644 --- a/titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java +++ b/titan-accumulo/titan-accumulo-core/src/test/java/com/thinkaurelius/titan/AccumuloStorageSetup.java @@ -47,7 +47,7 @@ public static Configuration getAccumuloGraphConfiguration() { storageConfig.addProperty(GraphDatabaseConfiguration.AUTH_USERNAME_KEY, "root"); storageConfig.addProperty(GraphDatabaseConfiguration.AUTH_PASSWORD_KEY, ""); - Configuration accumuloConfig = storageConfig.subset(AccumuloStoreManager.ACCUMULO_CONFIGURATION_NAMESPACE); + Configuration accumuloConfig = storageConfig.subset(AccumuloStoreManager.ACCUMULO_NAMESPACE); accumuloConfig.addProperty(AccumuloStoreManager.ACCUMULO_INTSANCE_KEY, "devdb"); accumuloConfig.addProperty(AccumuloStoreManager.SERVER_SIDE_ITERATORS_KEY, false); From 11a1abe1d73f23178394454cc35c22793a611755 Mon Sep 17 00:00:00 2001 From: Etienne Deprit Date: Wed, 15 Jan 2014 13:54:52 -0500 Subject: [PATCH 50/50] Move up Titan Accumulo version to 0.4.3-SNAPSHOT. --- titan-accumulo/pom.xml | 2 +- titan-accumulo/titan-accumulo-core/pom.xml | 2 +- titan-accumulo/titan-accumulo-iterators/pom.xml | 2 +- titan-dist/titan-dist-accumulo/pom.xml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/titan-accumulo/pom.xml b/titan-accumulo/pom.xml index 168b00267c..245e2b2498 100644 --- a/titan-accumulo/pom.xml +++ b/titan-accumulo/pom.xml @@ -4,7 +4,7 @@ com.thinkaurelius.titan titan - 0.4.2 + 0.4.3-SNAPSHOT ../pom.xml titan-accumulo diff --git a/titan-accumulo/titan-accumulo-core/pom.xml b/titan-accumulo/titan-accumulo-core/pom.xml index 4ea734e57e..3f9f6689c0 100644 --- a/titan-accumulo/titan-accumulo-core/pom.xml +++ b/titan-accumulo/titan-accumulo-core/pom.xml @@ -4,7 +4,7 @@ com.thinkaurelius.titan titan-accumulo - 0.4.2 + 0.4.3-SNAPSHOT ../pom.xml titan-accumulo-core diff --git a/titan-accumulo/titan-accumulo-iterators/pom.xml b/titan-accumulo/titan-accumulo-iterators/pom.xml index 9c57d3f945..7233b44f66 100644 --- a/titan-accumulo/titan-accumulo-iterators/pom.xml +++ b/titan-accumulo/titan-accumulo-iterators/pom.xml @@ -4,7 +4,7 @@ com.thinkaurelius.titan titan-accumulo - 0.4.2 + 0.4.3-SNAPSHOT ../pom.xml titan-accumulo-iterators diff --git a/titan-dist/titan-dist-accumulo/pom.xml b/titan-dist/titan-dist-accumulo/pom.xml index 0a70fcba28..73e2084d2a 100644 --- a/titan-dist/titan-dist-accumulo/pom.xml +++ b/titan-dist/titan-dist-accumulo/pom.xml @@ -3,7 +3,7 @@ com.thinkaurelius.titan titan-dist-parent - 0.4.2 + 0.4.3-SNAPSHOT ../titan-dist-parent/pom.xml pom