diff --git a/ChangeLog.md b/ChangeLog.md index 81a5708..32344fb 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,3 +1,7 @@ +## 0.0.14 +- Updated cassandra-unloader to add support for collections, + consistency level, ssl, etc + ## 0.0.13 - Added configFile - added ssl options (with truststore and keystore) diff --git a/README.md b/README.md index 278ec9b..54d30a5 100644 --- a/README.md +++ b/README.md @@ -15,11 +15,11 @@ loading of various types of delimited files, including ### Downloading This utility has already been built, and is available at -https://github.com/brianmhess/cassandra-loader/releases/download/v0.0.13/cassandra-loader +https://github.com/brianmhess/cassandra-loader/releases/download/v0.0.14/cassandra-loader Get it with wget: ``` -wget https://github.com/brianmhess/cassandra-loader/releases/download/v0.0.13/cassandra-loader +wget https://github.com/brianmhess/cassandra-loader/releases/download/v0.0.14/cassandra-loader ``` ### Building @@ -255,17 +255,24 @@ cassandra-unloader Usage statement: ``` +version: 0.0.14 Usage: -f -host -schema [OPTIONS] OPTIONS: + -configFile File with configuration options -delim Delimiter to use [,] -dateFormat Date format [default for Locale.ENGLISH] -nullString String that signifies NULL [none] -port CQL Port Number [9042] -user Cassandra username [none] -pw Password for user [none] + -ssl-truststore-path Path to SSL truststore [none] + -ssl-truststore-pw Password for SSL truststore [none] + -ssl-keystore-path Path to SSL keystore [none] + -ssl-keystore-pw Password for SSL keystore [none] + -consistencyLevel Consistency level [LOCAL_ONE] -decimalDelim Decimal delimiter [.] Other option is ',' -boolStyle Style for booleans [TRUE_FALSE] - -numThreads Number of concurrent threads (files) to load [5] + -numThreads Number of concurrent threads to unload [5] -beginToken Begin token [none] -endToken End token [none] ``` diff --git a/build.gradle b/build.gradle index 763923d..fae7358 100644 --- a/build.gradle +++ b/build.gradle @@ -1,7 +1,7 @@ apply plugin: 'java' apply plugin: 'application' -def versionNum = '0.0.13' +def versionNum = '0.0.14' allprojects { tasks.withType(JavaCompile) { diff --git a/src/main/java/com/datastax/loader/CqlDelimLoad.java b/src/main/java/com/datastax/loader/CqlDelimLoad.java index 2dc3199..090f666 100644 --- a/src/main/java/com/datastax/loader/CqlDelimLoad.java +++ b/src/main/java/com/datastax/loader/CqlDelimLoad.java @@ -86,7 +86,7 @@ import com.codahale.metrics.Timer; public class CqlDelimLoad { - private String version = "0.0.13"; + private String version = "0.0.14"; private String host = null; private int port = 9042; private String username = null; @@ -152,7 +152,7 @@ private String usage() { usage.append(" -ssl-truststore-pw Password for SSL truststore [none]\n"); usage.append(" -ssl-keystore-path Path to SSL keystore [none]\n"); usage.append(" -ssl-keystore-pw Password for SSL keystore [none]\n"); - usage.append(" -consistencyLevel Consistency level [LOCAL_ONE]"); + usage.append(" -consistencyLevel Consistency level [LOCAL_ONE]\n"); usage.append(" -numFutures Number of CQL futures to keep in flight [1000]\n"); usage.append(" -batchSize Number of INSERTs to batch together [1]\n"); usage.append(" -decimalDelim Decimal delimiter [.] Other option is ','\n"); diff --git a/src/main/java/com/datastax/loader/CqlDelimLoadTask.java b/src/main/java/com/datastax/loader/CqlDelimLoadTask.java index 7696750..2fe91da 100644 --- a/src/main/java/com/datastax/loader/CqlDelimLoadTask.java +++ b/src/main/java/com/datastax/loader/CqlDelimLoadTask.java @@ -165,7 +165,7 @@ private void setup() throws IOException, ParseException { cdp = new CqlDelimParser(cqlSchema, delimiter, nullString, dateFormatString, boolStyle, locale, - skipCols, session); + skipCols, session, true); insert = cdp.generateInsert(); statement = session.prepare(insert); statement.setRetryPolicy(new LoaderRetryPolicy(numRetries)); diff --git a/src/main/java/com/datastax/loader/CqlDelimParser.java b/src/main/java/com/datastax/loader/CqlDelimParser.java index db18402..0ecd850 100644 --- a/src/main/java/com/datastax/loader/CqlDelimParser.java +++ b/src/main/java/com/datastax/loader/CqlDelimParser.java @@ -61,10 +61,10 @@ public class CqlDelimParser { public CqlDelimParser(String inCqlSchema, String inDelimiter, String inNullString, String inDateFormatString, BooleanParser.BoolStyle inBoolStyle, Locale inLocale, - String skipList, Session session) + String skipList, Session session, boolean bLoader) throws ParseException { // Optionally provide things for the line parser - date format, boolean format, locale - initPmap(inDateFormatString, inBoolStyle, inLocale); + initPmap(inDateFormatString, inBoolStyle, inLocale, bLoader); processCqlSchema(inCqlSchema, session); createDelimParser(inDelimiter, inNullString, skipList); } @@ -78,12 +78,12 @@ private class SchemaBits { // intialize the Parsers and the parser map private void initPmap(String dateFormatString, BooleanParser.BoolStyle inBoolStyle, - Locale inLocale) { + Locale inLocale, boolean bLoader) { pmap = new HashMap(); - Parser integerParser = new IntegerParser(inLocale); - Parser longParser = new LongParser(inLocale); - Parser floatParser = new FloatParser(inLocale); - Parser doubleParser = new DoubleParser(inLocale); + Parser integerParser = new IntegerParser(inLocale, bLoader); + Parser longParser = new LongParser(inLocale, bLoader); + Parser floatParser = new FloatParser(inLocale, bLoader); + Parser doubleParser = new DoubleParser(inLocale, bLoader); Parser stringParser = new StringParser(); Parser booleanParser = new BooleanParser(inBoolStyle); Parser uuidParser = new UUIDParser(); diff --git a/src/main/java/com/datastax/loader/CqlDelimUnload.java b/src/main/java/com/datastax/loader/CqlDelimUnload.java index 0086792..93627eb 100644 --- a/src/main/java/com/datastax/loader/CqlDelimUnload.java +++ b/src/main/java/com/datastax/loader/CqlDelimUnload.java @@ -75,7 +75,7 @@ public class CqlDelimUnload { - private String version = "0.0.12"; + private String version = "0.0.14"; private String host = null; private int port = 9042; private String username = null; @@ -116,10 +116,10 @@ private String usage() { usage.append(" -ssl-truststore-pw Password for SSL truststore [none]\n"); usage.append(" -ssl-keystore-path Path to SSL keystore [none]\n"); usage.append(" -ssl-keystore-pw Password for SSL keystore [none]\n"); - usage.append(" -consistencyLevel Consistency level [LOCAL_ONE]"); + usage.append(" -consistencyLevel Consistency level [LOCAL_ONE]\n"); usage.append(" -decimalDelim Decimal delimiter [.] Other option is ','\n"); usage.append(" -boolStyle Style for booleans [TRUE_FALSE]\n"); - usage.append(" -numThreads Number of concurrent threads (files) to load [5]\n"); + usage.append(" -numThreads Number of concurrent threads to unload [5]\n"); usage.append(" -beginToken Begin token [none]\n"); usage.append(" -endToken End token [none]\n"); return usage.toString(); @@ -533,7 +533,7 @@ private String getPartitionKey(CqlDelimParser cdp, Session tsession) { private void setup() throws IOException, ParseException { cdp = new CqlDelimParser(cqlSchema, delimiter, nullString, dateFormatString, - boolStyle, locale, null, session); + boolStyle, locale, null, session, false); String select = cdp.generateSelect(); String partitionKey = getPartitionKey(cdp, session); if (null != beginToken) { diff --git a/src/main/java/com/datastax/loader/parser/DoubleParser.java b/src/main/java/com/datastax/loader/parser/DoubleParser.java index e37e4cb..d10e4c2 100644 --- a/src/main/java/com/datastax/loader/parser/DoubleParser.java +++ b/src/main/java/com/datastax/loader/parser/DoubleParser.java @@ -33,6 +33,10 @@ public DoubleParser() { public DoubleParser(Locale inLocale) { super(inLocale); } + + public DoubleParser(Locale inLocale, Boolean grouping) { + super(inLocale, grouping); + } public Double parse(String toparse) throws ParseException { Number val = super.parse(toparse); diff --git a/src/main/java/com/datastax/loader/parser/FloatParser.java b/src/main/java/com/datastax/loader/parser/FloatParser.java index 4b61c54..78c3cd2 100644 --- a/src/main/java/com/datastax/loader/parser/FloatParser.java +++ b/src/main/java/com/datastax/loader/parser/FloatParser.java @@ -34,6 +34,10 @@ public FloatParser(Locale inLocale) { super(inLocale); } + public FloatParser(Locale inLocale, Boolean grouping) { + super(inLocale, grouping); + } + public Float parse(String toparse) throws ParseException { Number val = super.parse(toparse); return (null == val) ? null : val.floatValue(); diff --git a/src/main/java/com/datastax/loader/parser/IntegerParser.java b/src/main/java/com/datastax/loader/parser/IntegerParser.java index f41af69..606eb21 100644 --- a/src/main/java/com/datastax/loader/parser/IntegerParser.java +++ b/src/main/java/com/datastax/loader/parser/IntegerParser.java @@ -34,6 +34,10 @@ public IntegerParser(Locale inLocale) { super(inLocale); } + public IntegerParser(Locale inLocale, Boolean grouping) { + super(inLocale, grouping); + } + public Integer parse(String toparse) throws ParseException { Number val = super.parse(toparse); return (null == val) ? null : val.intValue(); diff --git a/src/main/java/com/datastax/loader/parser/LongParser.java b/src/main/java/com/datastax/loader/parser/LongParser.java index b0b6e73..094b573 100644 --- a/src/main/java/com/datastax/loader/parser/LongParser.java +++ b/src/main/java/com/datastax/loader/parser/LongParser.java @@ -34,6 +34,10 @@ public LongParser(Locale inLocale) { super(inLocale); } + public LongParser(Locale inLocale, Boolean grouping) { + super(inLocale, grouping); + } + public Long parse(String toparse) throws ParseException { Number val = super.parse(toparse); return (null == val) ? null : val.longValue(); diff --git a/src/main/java/com/datastax/loader/parser/NumberParser.java b/src/main/java/com/datastax/loader/parser/NumberParser.java index 4bb0bb6..304990f 100644 --- a/src/main/java/com/datastax/loader/parser/NumberParser.java +++ b/src/main/java/com/datastax/loader/parser/NumberParser.java @@ -19,6 +19,7 @@ import java.lang.Number; import java.util.Locale; import java.text.NumberFormat; +import java.text.DecimalFormat; import java.text.ParseException; import java.lang.IndexOutOfBoundsException; import com.datastax.driver.core.Row; @@ -34,9 +35,16 @@ public NumberParser() { } public NumberParser(Locale locale) { + this(locale, true); + } + + public NumberParser(Locale locale, Boolean grouping) { if (null == locale) locale = Locale.ENGLISH; nf = NumberFormat.getInstance(locale); + if (nf instanceof DecimalFormat) { + ((DecimalFormat) nf).setGroupingUsed(grouping); + } } // Need this method for the subclasses