#9: streaming reading csv string data.

frictionlessdata · Nov 28, 2017 · 27681b2 · 27681b2
1 parent 9899fbe
commit 27681b2
Show file tree

Hide file tree

Showing 3 changed files with 75 additions and 16 deletions.
diff --git a/src/main/java/io/frictionlessdata/datapackage/Profile.java b/src/main/java/io/frictionlessdata/datapackage/Profile.java
@@ -6,6 +6,6 @@
  */
 public class Profile {
     public final static String PROFILE_DEFAULT = "data-package";
-    public static final String PROFILE_TABULAR_DATA_RESOURCE = "tabular-data-resource";
+    public final static String PROFILE_TABULAR_DATA_RESOURCE = "tabular-data-resource";
 
 }
diff --git a/src/main/java/io/frictionlessdata/datapackage/Resource.java b/src/main/java/io/frictionlessdata/datapackage/Resource.java
@@ -5,8 +5,11 @@
 import java.io.FileReader;
 import java.io.IOException;
 import java.io.Reader;
+import java.util.Iterator;
 import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVParser;
 import org.apache.commons.csv.CSVRecord;
+import org.json.JSONArray;
 
 /**
  * Resource.
@@ -35,6 +38,9 @@ public class Resource {
     private String hash = null;
 
     // hashes and licenes?
+
+    public final static String FORMAT_CSV = "csv";
+    public final static String FORMAT_JSON = "json";
 
     public Resource(String name, String path){
         this.name = name;
@@ -47,20 +53,29 @@ public Resource(String name, Object data, String format){
         this.format = format;
     }
 
-    public Iterable<CSVRecord> iter() throws IOException, FileNotFoundException, DataPackageException{
+    public Iterator<CSVRecord> iter() throws IOException, FileNotFoundException, DataPackageException{
         // Error for non tabular
         if(!this.profile.equalsIgnoreCase(Profile.PROFILE_TABULAR_DATA_RESOURCE)){
             throw new DataPackageException("Unsupported for non tabular data.");
         }
 
         if(this.path != null){
             Reader in = new FileReader(this.path);
-            Iterable<CSVRecord> csvRecords = CSVFormat.RFC4180.parse(in);
-
-            return csvRecords;
+            return CSVFormat.RFC4180.parse(in).iterator();
 
         }else if (this.data != null){
-            return null;
+            if(this.data instanceof String){
+                CSVParser parser = CSVParser.parse((String)this.data, CSVFormat.RFC4180);
+                return parser.getRecords().iterator();
+
+            }else if(this.data instanceof JSONArray){
+                //TODO: Implement: 
+                JSONArray dataArr = (JSONArray)this.data;
+                return null;
+
+            }else{
+                throw new DataPackageException("A resource has an invalid data format. It should be a CSV String or a JSON Array.");
+            }
 
         }else{
             throw new DataPackageException("No data has been set.");

diff --git a/src/test/java/io/frictionlessdata/datapackage/ResourceTest.java b/src/test/java/io/frictionlessdata/datapackage/ResourceTest.java
@@ -4,6 +4,7 @@
 import java.io.IOException;
 import java.net.MalformedURLException;
 import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.List;
 import org.apache.commons.csv.CSVRecord;
 import org.junit.Assert;
@@ -16,7 +17,7 @@
 public class ResourceTest {
 
     @Test
-    public void testValidationWithValidProfileUrl() throws IOException, DataPackageException, MalformedURLException{
+    public void testIterDataFromPath() throws IOException, DataPackageException, MalformedURLException{
 
         String filePath = ResourceTest.class.getResource("/fixtures/data/population.csv").getPath();
         Resource resource = new Resource("population", filePath);
@@ -25,18 +26,47 @@ public void testValidationWithValidProfileUrl() throws IOException, DataPackageE
         resource.setProfile(Profile.PROFILE_TABULAR_DATA_RESOURCE);
 
         // Expected data.
-        List<String[]> expectedData = new ArrayList();
-        expectedData.add(new String[]{"city", "year", "population"});
-        expectedData.add(new String[]{"london", "2017", "8780000"});
-        expectedData.add(new String[]{"paris", "2017", "2240000"});
-        expectedData.add(new String[]{"rome", "2017", "2860000"});
-
+        List<String[]> expectedData = this.getExpectedPopulationData();
+
         // Get iterator.
-        Iterable<CSVRecord> records = resource.iter();
+        Iterator<CSVRecord> iter = resource.iter();
+        int expectedDataIndex = 0;
+
+        // Assert data.
+        while(iter.hasNext()){
+            CSVRecord record = iter.next();
+            String city = record.get(0);
+            String year = record.get(1);
+            String population = record.get(2);
+
+            Assert.assertEquals(expectedData.get(expectedDataIndex)[0], city);
+            Assert.assertEquals(expectedData.get(expectedDataIndex)[1], year);
+            Assert.assertEquals(expectedData.get(expectedDataIndex)[2], population);
+
+            expectedDataIndex++;
+        }    
+    }
+
+    @Test
+    public void testIterDataFromDataInCsvForm() throws IOException, DataPackageException{
+        String filePath = ResourceTest.class.getResource("/fixtures/data/population.csv").getPath();
+
+        String dataString = "city,year,population\nlondon,2017,8780000\nparis,2017,2240000\nrome,2017,2860000";
+        Resource resource = new Resource("population", dataString, Resource.FORMAT_CSV);
+
+        // Set the profile to tabular data resource.
+        resource.setProfile(Profile.PROFILE_TABULAR_DATA_RESOURCE);
+
+        // Expected data.
+        List<String[]> expectedData = this.getExpectedPopulationData();
+
+        // Get Iterator.
+        Iterator<CSVRecord> iter = resource.iter();
         int expectedDataIndex = 0;
 
         // Assert data.
-        for (CSVRecord record : records) {
+        while(iter.hasNext()){
+            CSVRecord record = iter.next();
             String city = record.get(0);
             String year = record.get(1);
             String population = record.get(2);
@@ -46,8 +76,22 @@ public void testValidationWithValidProfileUrl() throws IOException, DataPackageE
             Assert.assertEquals(expectedData.get(expectedDataIndex)[2], population);
 
             expectedDataIndex++;
-        }
+        }  
+    }
+
+    @Test
+    public void testIterDataFromDataInJSONForm(){
+
+    }
+
+    private List<String[]> getExpectedPopulationData(){
+        List<String[]> expectedData  = new ArrayList();
+        expectedData.add(new String[]{"city", "year", "population"});
+        expectedData.add(new String[]{"london", "2017", "8780000"});
+        expectedData.add(new String[]{"paris", "2017", "2240000"});
+        expectedData.add(new String[]{"rome", "2017", "2860000"});
 
+        return expectedData;
     }
 
 }