From 94bc0a7bd897338e49545b74b5a96d0c21d69829 Mon Sep 17 00:00:00 2001 From: SriramKeerthi Date: Sun, 27 Nov 2016 14:12:01 -0800 Subject: [PATCH] Adding JSON Iterator and removing unnecessary imports --- .../caffinc/jaggr/utils/JsonFileReader.java | 6 +- .../com/caffinc/jaggr/utils/JsonIterator.java | 132 ++++++++++++++++++ .../caffinc/jaggr/utils/JsonIteratorTest.java | 50 +++++++ .../com/caffinc/jaggr/core/Aggregation.java | 6 +- 4 files changed, 188 insertions(+), 6 deletions(-) create mode 100644 jaggr/jaggr-utils/src/main/java/com/caffinc/jaggr/utils/JsonIterator.java create mode 100644 jaggr/jaggr-utils/src/test/java/com/caffinc/jaggr/utils/JsonIteratorTest.java diff --git a/jaggr/jaggr-utils/src/main/java/com/caffinc/jaggr/utils/JsonFileReader.java b/jaggr/jaggr-utils/src/main/java/com/caffinc/jaggr/utils/JsonFileReader.java index 76e75af..5d7bcbc 100644 --- a/jaggr/jaggr-utils/src/main/java/com/caffinc/jaggr/utils/JsonFileReader.java +++ b/jaggr/jaggr-utils/src/main/java/com/caffinc/jaggr/utils/JsonFileReader.java @@ -27,7 +27,7 @@ public class JsonFileReader { * @return List of JSON lines from the file * @throws IOException thrown if there is a problem accessing the file */ - public static List> readJsonFromResource(String resourceName) throws IOException { + public static List> readJsonFromResource(final String resourceName) throws IOException { List> jsonList = new ArrayList<>(); String json; try (BufferedReader br = new BufferedReader( @@ -46,7 +46,7 @@ public static List> readJsonFromResource(String resourceName * @return List of JSON lines from the file * @throws IOException thrown if there is a problem accessing the file */ - public static List> readJsonFromFile(String fileName) throws IOException { + public static List> readJsonFromFile(final String fileName) throws IOException { List> jsonList = new ArrayList<>(); for (String json : getFileLines(fileName)) { jsonList.add(gson.fromJson(json, HashMap.class)); @@ -61,7 +61,7 @@ public static List> readJsonFromFile(String fileName) throws * @return List of lines from the file * @throws IOException thrown if there is a problem accessing the file */ - public static List getFileLines(String fileName) throws IOException { + public static List getFileLines(final String fileName) throws IOException { List text = new ArrayList<>(); try (BufferedReader br = new BufferedReader(new FileReader(fileName))) { String line; diff --git a/jaggr/jaggr-utils/src/main/java/com/caffinc/jaggr/utils/JsonIterator.java b/jaggr/jaggr-utils/src/main/java/com/caffinc/jaggr/utils/JsonIterator.java new file mode 100644 index 0000000..ae533e8 --- /dev/null +++ b/jaggr/jaggr-utils/src/main/java/com/caffinc/jaggr/utils/JsonIterator.java @@ -0,0 +1,132 @@ +package com.caffinc.jaggr.utils; + +import com.google.gson.Gson; + +import java.io.*; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.NoSuchElementException; + +/** + * Iterates a JSON file + * + * @author Sriram + * @since 11/27/2016 + */ +public class JsonIterator implements Iterator>, Closeable { + private final BufferedReader bufferedReader; + private String cachedLine; + private boolean finished = false; + private Gson gson = new Gson(); + + /** + * Constructs an iterator of the lines for a fileName. + * + * @param fileName the fileName to read from + * @throws IOException thrown if there is a problem accessing the file + */ + public JsonIterator(final String fileName) throws IOException { + this(new BufferedReader(new FileReader(fileName))); + } + + /** + * Constructs an iterator of the lines for a Reader. + * + * @param reader the Reader to read from, not null + * @throws IllegalArgumentException if the reader is null + */ + public JsonIterator(final Reader reader) throws IllegalArgumentException { + if (reader == null) { + throw new IllegalArgumentException("Reader must not be null"); + } + if (reader instanceof BufferedReader) { + bufferedReader = (BufferedReader) reader; + } else { + bufferedReader = new BufferedReader(reader); + } + } + + /** + * Indicates whether the Reader has more lines. + * If there is an IOException then {@link #close()} will + * be called on this instance. + * + * @return {@code true} if the Reader has more lines + * @throws IllegalStateException if an IO exception occurs + */ + public boolean hasNext() { + if (cachedLine != null) { + return true; + } else if (finished) { + return false; + } else { + try { + while (true) { + final String line = bufferedReader.readLine(); + if (line == null) { + finished = true; + return false; + } + cachedLine = line; + return true; + } + } catch (final IOException ioe) { + close(); + throw new IllegalStateException(ioe); + } + } + } + + /** + * Returns the next object in the file or wrapped Reader. + * + * @return the next JSON object from the input + * @throws NoSuchElementException if there is no object to return + */ + public Map next() { + return nextObject(); + } + + /** + * Returns the next object in the file or wrapped Reader. + * + * @return the next JSON object from the input + * @throws NoSuchElementException if there is no object to return + */ + public Map nextObject() { + if (!hasNext()) { + throw new NoSuchElementException("No more objects"); + } + final String currentLine = cachedLine; + cachedLine = null; + return gson.fromJson(currentLine, HashMap.class); + } + + /** + * Closes the underlying Reader quietly. + * This method is useful if you only want to process the first few + * lines of a larger file. If you do not close the iterator + * then the Reader remains open. + * This method can safely be called multiple times. + */ + public void close() { + finished = true; + try { + bufferedReader.close(); + } catch (final IOException ioe) { + // ignore + } + cachedLine = null; + } + + /** + * Unsupported. + * + * @throws UnsupportedOperationException always + */ + public void remove() { + throw new UnsupportedOperationException("Remove unsupported on JsonIterator"); + } + +} diff --git a/jaggr/jaggr-utils/src/test/java/com/caffinc/jaggr/utils/JsonIteratorTest.java b/jaggr/jaggr-utils/src/test/java/com/caffinc/jaggr/utils/JsonIteratorTest.java new file mode 100644 index 0000000..42d48de --- /dev/null +++ b/jaggr/jaggr-utils/src/test/java/com/caffinc/jaggr/utils/JsonIteratorTest.java @@ -0,0 +1,50 @@ +package com.caffinc.jaggr.utils; + +import com.google.gson.Gson; +import org.junit.Assert; +import org.junit.Test; + +import java.io.BufferedWriter; +import java.io.FileWriter; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; + +/** + * Tests for the JsonIterator + * + * @author Sriram + * @since 11/27/2016 + */ +public class JsonIteratorTest { + private static final String TEMP_DIR = System.getProperty("java.io.tmpdir"); + private static final Random RANDOM = new Random(); + private static final Gson GSON = new Gson(); + + @Test + public void testJsonFileIterator() throws Exception { + Path tempFilePath = Paths.get(TEMP_DIR, "jsontest" + RANDOM.nextInt() + ".json"); + try { + List> expectedData = new ArrayList<>(); + try (BufferedWriter br = new BufferedWriter(new FileWriter(tempFilePath.toFile())) + ) { + for (int i = 0; i < 10; i++) { + Map json = new HashMap<>(); + json.put("_id", (double) i); + json.put("val", RANDOM.nextDouble()); + expectedData.add(json); + br.write(GSON.toJson(json) + "\n"); + } + } + try (JsonIterator jsonIterator = new JsonIterator(tempFilePath.toString())) { + for (Map expected : expectedData) { + Map actual = jsonIterator.next(); + Assert.assertEquals("Value should match value written to file", expected, actual); + } + } + } finally { + Files.delete(tempFilePath); + } + } +} diff --git a/jaggr/jaggr/src/main/java/com/caffinc/jaggr/core/Aggregation.java b/jaggr/jaggr/src/main/java/com/caffinc/jaggr/core/Aggregation.java index c2a3011..f61cbe6 100644 --- a/jaggr/jaggr/src/main/java/com/caffinc/jaggr/core/Aggregation.java +++ b/jaggr/jaggr/src/main/java/com/caffinc/jaggr/core/Aggregation.java @@ -40,12 +40,12 @@ public List> aggregate(Iterator> objectI } /** - * Aggregates over a list of JSON Objects + * Aggregates over an iterable list of JSON Objects * - * @param objectList JSON Object list + * @param objectList Iterable list of JSON Objects * @return aggregation result */ - public List> aggregate(List> objectList) { + public List> aggregate(Iterable> objectList) { Map> workspace = new HashMap<>(); if (objectList != null) { for (Map object : objectList) {